From 7768ff1e414d8fa4aa02b888b9370c34f5e62de1 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Wed, 8 May 2024 16:57:05 +0200 Subject: [PATCH 001/903] Python 3.14.0a0 --- Doc/tutorial/interpreter.rst | 6 +- Doc/tutorial/stdlib.rst | 2 +- Doc/tutorial/stdlib2.rst | 2 +- Doc/whatsnew/3.14 | 131 +++++++++++++++++++++++++++++++++++ Include/patchlevel.h | 8 +-- README.rst | 4 +- configure.ac | 2 +- 7 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 Doc/whatsnew/3.14 diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 299b6c2777adc0..02e7de77322e99 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -16,7 +16,7 @@ Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.13 + python3.14 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.13 - Python 3.13 (default, April 4 2023, 09:25:04) + $ python3.14 + Python 3.14 (default, April 4 2024, 09:25:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 9def2a5714950b..86c94429339dfc 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python313' + 'C:\\Python314' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 09b6f3d91bcfed..4bc810ce36c71b 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -279,7 +279,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python313/lib/weakref.py", line 46, in __getitem__ + File "C:/python314/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' diff --git a/Doc/whatsnew/3.14 b/Doc/whatsnew/3.14 new file mode 100644 index 00000000000000..14628f666dd079 --- /dev/null +++ b/Doc/whatsnew/3.14 @@ -0,0 +1,131 @@ + +**************************** + What's New In Python 3.14 +**************************** + +:Editor: TBD + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.14, compared to 3.13. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.14 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.14. + Brevity is key. + + +.. PEP-sized items next. + + + +New Features +============ + + + +Other Language Changes +====================== + + + +New Modules +=========== + +* None yet. + + +Improved Modules +================ + + +Optimizations +============= + + + + +Deprecated +========== + + + +Removed +======= + + + +Porting to Python 3.14 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Build Changes +============= + + +C API Changes +============= + +New Features +------------ + +Porting to Python 3.14 +---------------------- + +Deprecated +---------- + +Removed +------- + diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 179c9f50214935..d63af11dbd220a 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -17,13 +17,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 13 +#define PY_MINOR_VERSION 14 #define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.13.0b1" +#define PY_VERSION "3.14.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/README.rst b/README.rst index ba8ba82ed87e3f..44b020e0c7ed5f 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.13.0 beta 1 -==================================== +This is Python version 3.14.0 alpha 0 +===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg :alt: CPython build status on GitHub Actions diff --git a/configure.ac b/configure.ac index 8eb96767487592..0f1b977591c260 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ dnl to regenerate the configure script. dnl # Set VERSION so we only need to edit in one place (i.e., here) -m4_define([PYTHON_VERSION], [3.13]) +m4_define([PYTHON_VERSION], [3.14]) AC_PREREQ([2.71]) From 891fd12e52d5963a338b30ce2cb7ccac72f8eea5 Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Wed, 8 May 2024 17:29:50 +0200 Subject: [PATCH 002/903] Regenerate configure for 3.14, which the release script forgot. (#118765) --- configure | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/configure b/configure index de426e6b686e68..c4b61fb8c1cfea 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for python 3.13. +# Generated by GNU Autoconf 2.71 for python 3.14. # # Report bugs to . # @@ -611,8 +611,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='python' PACKAGE_TARNAME='python' -PACKAGE_VERSION='3.13' -PACKAGE_STRING='python 3.13' +PACKAGE_VERSION='3.14' +PACKAGE_STRING='python 3.14' PACKAGE_BUGREPORT='https://github.com/python/cpython/issues/' PACKAGE_URL='' @@ -1723,7 +1723,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures python 3.13 to adapt to many kinds of systems. +\`configure' configures python 3.14 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1789,7 +1789,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of python 3.13:";; + short | recursive ) echo "Configuration of python 3.14:";; esac cat <<\_ACEOF @@ -1838,9 +1838,9 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-build-python=python3.13 + --with-build-python=python3.14 path to build python binary for cross compiling - (default: _bootstrap_python or python3.13) + (default: _bootstrap_python or python3.14) --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) @@ -2082,7 +2082,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -python configure 3.13 +python configure 3.14 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -2739,7 +2739,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by python $as_me 3.13, which was +It was created by python $as_me 3.14, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -3826,7 +3826,7 @@ rm confdefs.h mv confdefs.h.new confdefs.h -VERSION=3.13 +VERSION=3.14 # Version number of Python's own shared library file. @@ -32370,7 +32370,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by python $as_me 3.13, which was +This file was extended by python $as_me 3.14, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -32434,7 +32434,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -python config.status 3.13 +python config.status 3.14 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" From 66f8bb76a15e64a1bb7688b177ed29e26230fdee Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 8 May 2024 16:52:39 +0100 Subject: [PATCH 003/903] gh-118486: Update docs for CVE-2024-4030 reference (GH-118737) --- Doc/whatsnew/3.13.rst | 14 ++++++++++++++ .../2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst | 4 ++++ 2 files changed, 18 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 69264a3672bbbf..8e90faee667ded 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -847,6 +847,12 @@ os :c:func:`!posix_spawn_file_actions_addclosefrom_np`. (Contributed by Jakub Kulik in :gh:`113117`.) +* :func:`os.mkdir` and :func:`os.makedirs` on Windows now support passing a + *mode* value of ``0o700`` to apply access control to the new directory. This + implicitly affects :func:`tempfile.mkdtemp` and is a mitigation for + :cve:`2024-4030`. Other values for *mode* continue to be ignored. + (Contributed by Steve Dower in :gh:`118486`.) + os.path ------- @@ -989,6 +995,14 @@ sys This function is not guaranteed to exist in all implementations of Python. (Contributed by Serhiy Storchaka in :gh:`78573`.) +tempfile +-------- + +* On Windows, the default mode ``0o700`` used by :func:`tempfile.mkdtemp` now + limits access to the new directory due to changes to :func:`os.mkdir`. This + is a mitigation for :cve:`2024-4030`. + (Contributed by Steve Dower in :gh:`118486`.) + time ---- diff --git a/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst b/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst new file mode 100644 index 00000000000000..8ac48aac816a60 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst @@ -0,0 +1,4 @@ +:func:`os.mkdir` on Windows now accepts *mode* of ``0o700`` to restrict +the new directory to the current user. This fixes :cve:`2024-4030` +affecting :func:`tempfile.mkdtemp` in scenarios where the base temporary +directory is more permissive than the default. From ed2b0fb04474725a38312784e1695a1cd7c0cce1 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 8 May 2024 19:00:38 +0300 Subject: [PATCH 004/903] Update Windows library names for the Python version bump (#118766) --- PC/pyconfig.h.in | 8 ++++---- PCbuild/rt.bat | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h.in index d72d6282c2806f..f44e41c2e72f84 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h.in @@ -316,19 +316,19 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ file in their Makefile */ # if defined(Py_GIL_DISABLED) # if defined(_DEBUG) -# pragma comment(lib,"python313t_d.lib") +# pragma comment(lib,"python314t_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3t.lib") # else -# pragma comment(lib,"python313t.lib") +# pragma comment(lib,"python314t.lib") # endif /* _DEBUG */ # else /* Py_GIL_DISABLED */ # if defined(_DEBUG) -# pragma comment(lib,"python313_d.lib") +# pragma comment(lib,"python314_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3.lib") # else -# pragma comment(lib,"python313.lib") +# pragma comment(lib,"python314.lib") # endif /* _DEBUG */ # endif /* Py_GIL_DISABLED */ # endif /* _MSC_VER */ diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index ac530a5206271f..c436215780fda0 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -42,7 +42,7 @@ if "%~1"=="-O" (set dashO=-O) & shift & goto CheckOpts if "%~1"=="-q" (set qmode=yes) & shift & goto CheckOpts if "%~1"=="-d" (set suffix=_d) & shift & goto CheckOpts rem HACK: Need some way to infer the version number in this script -if "%~1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts +if "%~1"=="--disable-gil" (set pyname=python3.14t) & shift & goto CheckOpts if "%~1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts if "%~1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts if "%~1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts From 6d419db10c84cacbb3862e2adc940342175bfce9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 8 May 2024 09:40:40 -0700 Subject: [PATCH 005/903] Fix file extensions for 3.14 Whats New (#118770) --- Doc/whatsnew/{3.14 => 3.14.rst} | 0 Doc/whatsnew/index.rst | 1 + 2 files changed, 1 insertion(+) rename Doc/whatsnew/{3.14 => 3.14.rst} (100%) diff --git a/Doc/whatsnew/3.14 b/Doc/whatsnew/3.14.rst similarity index 100% rename from Doc/whatsnew/3.14 rename to Doc/whatsnew/3.14.rst diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index 39837f8c62548f..6ff722a1894585 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.14.rst 3.13.rst 3.12.rst 3.11.rst From aac6b019fe91e2f9f7a955d4fc4db5d5efd968c9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 8 May 2024 09:54:51 -0700 Subject: [PATCH 006/903] gh-118772: Allow TypeVars without a default to follow those with a default when constructing aliases (#118774) --- Lib/test/test_typing.py | 17 +++++++++++++ Lib/typing.py | 25 +++++++++++-------- ...-05-08-09-21-49.gh-issue-118772.c16E8X.rst | 2 ++ 3 files changed, 34 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-08-09-21-49.gh-issue-118772.c16E8X.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index bd116bb1ab7213..fff81f7997764d 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -668,6 +668,23 @@ class X(Generic[*Ts, T]): ... with self.assertRaises(TypeError): class Y(Generic[*Ts_default, T]): ... + def test_allow_default_after_non_default_in_alias(self): + T_default = TypeVar('T_default', default=int) + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + + a1 = Callable[[T_default], T] + self.assertEqual(a1.__args__, (T_default, T)) + + a2 = dict[T_default, T] + self.assertEqual(a2.__args__, (T_default, T)) + + a3 = typing.Dict[T_default, T] + self.assertEqual(a3.__args__, (T_default, T)) + + a4 = Callable[*Ts, T] + self.assertEqual(a4.__args__, (*Ts, T)) + def test_paramspec_specialization(self): T = TypeVar("T") P = ParamSpec('P', default=[str, int]) diff --git a/Lib/typing.py b/Lib/typing.py index 8e61f50477bcc2..c8649e312ddb00 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -257,7 +257,7 @@ def _type_repr(obj): return repr(obj) -def _collect_parameters(args): +def _collect_parameters(args, *, enforce_default_ordering: bool = True): """Collect all type variables and parameter specifications in args in order of first appearance (lexicographic order). @@ -286,15 +286,16 @@ def _collect_parameters(args): parameters.append(collected) elif hasattr(t, '__typing_subst__'): if t not in parameters: - if type_var_tuple_encountered and t.has_default(): - raise TypeError('Type parameter with a default' - ' follows TypeVarTuple') + if enforce_default_ordering: + if type_var_tuple_encountered and t.has_default(): + raise TypeError('Type parameter with a default' + ' follows TypeVarTuple') - if t.has_default(): - default_encountered = True - elif default_encountered: - raise TypeError(f'Type parameter {t!r} without a default' - ' follows type parameter with a default') + if t.has_default(): + default_encountered = True + elif default_encountered: + raise TypeError(f'Type parameter {t!r} without a default' + ' follows type parameter with a default') parameters.append(t) else: @@ -1416,7 +1417,11 @@ def __init__(self, origin, args, *, inst=True, name=None): args = (args,) self.__args__ = tuple(... if a is _TypingEllipsis else a for a in args) - self.__parameters__ = _collect_parameters(args) + enforce_default_ordering = origin in (Generic, Protocol) + self.__parameters__ = _collect_parameters( + args, + enforce_default_ordering=enforce_default_ordering, + ) if not name: self.__module__ = origin.__module__ diff --git a/Misc/NEWS.d/next/Library/2024-05-08-09-21-49.gh-issue-118772.c16E8X.rst b/Misc/NEWS.d/next/Library/2024-05-08-09-21-49.gh-issue-118772.c16E8X.rst new file mode 100644 index 00000000000000..474454b36da956 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-09-21-49.gh-issue-118772.c16E8X.rst @@ -0,0 +1,2 @@ +Allow :class:`typing.TypeVar` instances without a default to follow +instances without a default in some cases. Patch by Jelle Zijlstra. From 3c079a020369173745a16ed4ef19a64f69e8592b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 8 May 2024 11:12:00 -0700 Subject: [PATCH 007/903] gh-118767: Make bool(NotImplemented) raise TypeError (#118775) --- Doc/library/constants.rst | 3 +++ Doc/reference/datamodel.rst | 3 +++ Doc/whatsnew/3.14.rst | 3 +++ Lib/test/test_builtin.py | 11 ++++------- .../2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst | 2 ++ Objects/object.c | 10 +++------- 6 files changed, 18 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 93a7244f87de6b..890517c3eb68dc 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -57,6 +57,9 @@ A small number of constants live in the built-in namespace. They are: it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. It will raise a :exc:`TypeError` in a future version of Python. + .. versionchanged:: 3.14 + Evaluating :data:`!NotImplemented` in a boolean context now raises a :exc:`TypeError`. + .. index:: single: ...; ellipsis literal .. data:: Ellipsis diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index f5e87160732056..fc072b4e75314d 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -174,6 +174,9 @@ for more details. it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. It will raise a :exc:`TypeError` in a future version of Python. +.. versionchanged:: 3.14 + Evaluating :data:`NotImplemented` in a boolean context now raises a :exc:`TypeError`. + Ellipsis -------- diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 14628f666dd079..6a9d0b0d912fb2 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -101,6 +101,9 @@ Deprecated Removed ======= +* Using :data:`NotImplemented` in a boolean context will now raise a :exc:`TypeError`. + It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed + by Jelle Zijlstra in :gh:`118767`.) Porting to Python 3.14 diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 230789f29ff788..120814379dd53a 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2130,14 +2130,11 @@ def test_warning_notimplemented(self): # be evaluated in a boolean context (virtually all such use cases # are a result of accidental misuse implementing rich comparison # operations in terms of one another). - # For the time being, it will continue to evaluate as a true value, but - # issue a deprecation warning (with the eventual intent to make it - # a TypeError). - self.assertWarns(DeprecationWarning, bool, NotImplemented) - with self.assertWarns(DeprecationWarning): + self.assertRaises(TypeError, bool, NotImplemented) + with self.assertRaises(TypeError): self.assertTrue(NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertFalse(not NotImplemented) + with self.assertRaises(TypeError): + not NotImplemented class TestBreakpoint(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst new file mode 100644 index 00000000000000..76548effd1449f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst @@ -0,0 +1,2 @@ +Using :data:`NotImplemented` in a boolean context now raises +:exc:`TypeError`. Contributed by Jelle Zijlstra in :gh:`118767`. diff --git a/Objects/object.c b/Objects/object.c index 8ad0389cbc7626..29183dcc5b4a01 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2090,13 +2090,9 @@ notimplemented_dealloc(PyObject *notimplemented) static int notimplemented_bool(PyObject *v) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "NotImplemented should not be used in a boolean context", - 1) < 0) - { - return -1; - } - return 1; + PyErr_SetString(PyExc_TypeError, + "NotImplemented should not be used in a boolean context"); + return -1; } static PyNumberMethods notimplemented_as_number = { From c68311df8543384e04fe994b3d4f4718cca1040e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 8 May 2024 11:12:36 -0700 Subject: [PATCH 008/903] Run CI on the 3.13 branch (#118779) --- .github/workflows/build.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1b2677ff6fe889..a53f1ae1a46fc1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,19 +8,11 @@ on: push: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' + - '3.*' pull_request: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' + - '3.*' permissions: contents: read From 7b0c247f1c176e092777fce4677a00f22c738b3c Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Thu, 9 May 2024 02:20:40 +0800 Subject: [PATCH 009/903] Docs: fix typos in documentation (#118752) --- Doc/library/email.message.rst | 2 +- Doc/library/idle.rst | 2 +- Doc/library/pyexpat.rst | 2 +- Doc/library/sys.monitoring.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index adea067e082615..c10e1b50d4903b 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -41,7 +41,7 @@ The :class:`EmailMessage` dictionary-like interface is indexed by the header names, which must be ASCII values. The values of the dictionary are strings with some extra methods. Headers are stored and returned in case-preserving form, but field names are matched case-insensitively. The keys are ordered, -but unlike a real dict, there can be duplicates. Addtional methods are +but unlike a real dict, there can be duplicates. Additional methods are provided for working with headers that have duplicate keys. The *payload* is either a string or bytes object, in the case of simple message diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 17a5144b4c0635..59b181aab3e484 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -429,7 +429,7 @@ Several non-character keys move the cursor and possibly delete characters. Deletion does not puts text on the clipboard, but IDLE has an undo list. Wherever this doc discusses keys, 'C' refers to the :kbd:`Control` key on Windows and -Unix and the :kbd:`Command` key on macOS. (And all such dicussions +Unix and the :kbd:`Command` key on macOS. (And all such discussions assume that the keys have not been re-bound to something else.) * Arrow keys move the cursor one character or line. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index c4b4e6319277af..2c048b8290d014 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -210,7 +210,7 @@ XMLParser Objects by default until a sufficient amount of input is reached. Due to this delay, registered handlers may — depending of the sizing of input chunks pushed to Expat — no longer be called right after pushing new - input to the parser. Where immediate feedback and taking over responsiblity + input to the parser. Where immediate feedback and taking over responsibility of protecting against denial of service from large tokens are both wanted, calling ``SetReparseDeferralEnabled(False)`` disables reparse deferral for the current Expat parser instance, temporarily or altogether. diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 0e0095e108e9c0..583266daeb56a1 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -160,7 +160,7 @@ events, use the expression ``PY_RETURN | PY_START``. .. monitoring-event:: NO_EVENTS - An alias for ``0`` so users can do explict comparisions like:: + An alias for ``0`` so users can do explicit comparisons like:: if get_events(DEBUGGER_ID) == NO_EVENTS: ... From bcb435ee8ff41b5ec5d879ee0b6651f146a66151 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 8 May 2024 15:34:40 -0400 Subject: [PATCH 010/903] docs: module page titles should not start with a link to themselves (#117099) --- Doc/library/__future__.rst | 4 ++-- Doc/library/__main__.rst | 4 ++-- Doc/library/_thread.rst | 4 ++-- Doc/library/abc.rst | 4 ++-- Doc/library/argparse.rst | 4 ++-- Doc/library/array.rst | 4 ++-- Doc/library/ast.rst | 4 ++-- Doc/library/asyncio.rst | 4 ++-- Doc/library/atexit.rst | 4 ++-- Doc/library/base64.rst | 4 ++-- Doc/library/bdb.rst | 4 ++-- Doc/library/binascii.rst | 4 ++-- Doc/library/bisect.rst | 4 ++-- Doc/library/builtins.rst | 4 ++-- Doc/library/bz2.rst | 4 ++-- Doc/library/calendar.rst | 4 ++-- Doc/library/cmath.rst | 4 ++-- Doc/library/cmd.rst | 4 ++-- Doc/library/code.rst | 4 ++-- Doc/library/codecs.rst | 4 ++-- Doc/library/codeop.rst | 4 ++-- Doc/library/collections.abc.rst | 4 ++-- Doc/library/collections.rst | 4 ++-- Doc/library/colorsys.rst | 4 ++-- Doc/library/compileall.rst | 4 ++-- Doc/library/concurrent.futures.rst | 4 ++-- Doc/library/configparser.rst | 4 ++-- Doc/library/contextvars.rst | 4 ++-- Doc/library/copy.rst | 4 ++-- Doc/library/copyreg.rst | 4 ++-- Doc/library/csv.rst | 4 ++-- Doc/library/ctypes.rst | 4 ++-- Doc/library/curses.ascii.rst | 4 ++-- Doc/library/curses.panel.rst | 4 ++-- Doc/library/curses.rst | 4 ++-- Doc/library/datetime.rst | 4 ++-- Doc/library/dbm.rst | 4 ++-- Doc/library/decimal.rst | 4 ++-- Doc/library/difflib.rst | 4 ++-- Doc/library/dis.rst | 4 ++-- Doc/library/doctest.rst | 4 ++-- Doc/library/email.charset.rst | 4 ++-- Doc/library/email.contentmanager.rst | 4 ++-- Doc/library/email.encoders.rst | 4 ++-- Doc/library/email.errors.rst | 4 ++-- Doc/library/email.generator.rst | 4 ++-- Doc/library/email.header.rst | 4 ++-- Doc/library/email.headerregistry.rst | 4 ++-- Doc/library/email.iterators.rst | 4 ++-- Doc/library/email.message.rst | 4 ++-- Doc/library/email.mime.rst | 4 ++-- Doc/library/email.parser.rst | 4 ++-- Doc/library/email.policy.rst | 4 ++-- Doc/library/email.rst | 4 ++-- Doc/library/email.utils.rst | 4 ++-- Doc/library/ensurepip.rst | 4 ++-- Doc/library/enum.rst | 4 ++-- Doc/library/errno.rst | 4 ++-- Doc/library/faulthandler.rst | 4 ++-- Doc/library/fcntl.rst | 4 ++-- Doc/library/filecmp.rst | 4 ++-- Doc/library/fileinput.rst | 4 ++-- Doc/library/fnmatch.rst | 4 ++-- Doc/library/fractions.rst | 4 ++-- Doc/library/ftplib.rst | 4 ++-- Doc/library/functools.rst | 4 ++-- Doc/library/gc.rst | 4 ++-- Doc/library/getopt.rst | 4 ++-- Doc/library/getpass.rst | 4 ++-- Doc/library/gettext.rst | 4 ++-- Doc/library/glob.rst | 4 ++-- Doc/library/graphlib.rst | 4 ++-- Doc/library/grp.rst | 4 ++-- Doc/library/gzip.rst | 4 ++-- Doc/library/hashlib.rst | 4 ++-- Doc/library/heapq.rst | 4 ++-- Doc/library/hmac.rst | 4 ++-- Doc/library/html.entities.rst | 4 ++-- Doc/library/html.parser.rst | 4 ++-- Doc/library/html.rst | 4 ++-- Doc/library/http.client.rst | 4 ++-- Doc/library/http.cookiejar.rst | 4 ++-- Doc/library/http.cookies.rst | 4 ++-- Doc/library/http.rst | 4 ++-- Doc/library/http.server.rst | 4 ++-- Doc/library/imaplib.rst | 4 ++-- Doc/library/importlib.resources.abc.rst | 4 ++-- Doc/library/importlib.resources.rst | 4 ++-- Doc/library/inspect.rst | 4 ++-- Doc/library/io.rst | 4 ++-- Doc/library/ipaddress.rst | 4 ++-- Doc/library/itertools.rst | 4 ++-- Doc/library/json.rst | 4 ++-- Doc/library/keyword.rst | 4 ++-- Doc/library/linecache.rst | 4 ++-- Doc/library/locale.rst | 4 ++-- Doc/library/logging.config.rst | 4 ++-- Doc/library/logging.handlers.rst | 4 ++-- Doc/library/logging.rst | 4 ++-- Doc/library/lzma.rst | 4 ++-- Doc/library/mailbox.rst | 4 ++-- Doc/library/marshal.rst | 4 ++-- Doc/library/math.rst | 4 ++-- Doc/library/mimetypes.rst | 4 ++-- Doc/library/mmap.rst | 4 ++-- Doc/library/modulefinder.rst | 4 ++-- Doc/library/msvcrt.rst | 4 ++-- Doc/library/multiprocessing.rst | 4 ++-- Doc/library/multiprocessing.shared_memory.rst | 4 ++-- Doc/library/netrc.rst | 5 ++--- Doc/library/numbers.rst | 4 ++-- Doc/library/operator.rst | 4 ++-- Doc/library/optparse.rst | 4 ++-- Doc/library/os.path.rst | 4 ++-- Doc/library/os.rst | 4 ++-- Doc/library/pathlib.rst | 5 ++--- Doc/library/pickle.rst | 4 ++-- Doc/library/pickletools.rst | 4 ++-- Doc/library/pkgutil.rst | 4 ++-- Doc/library/platform.rst | 4 ++-- Doc/library/plistlib.rst | 4 ++-- Doc/library/poplib.rst | 4 ++-- Doc/library/posix.rst | 4 ++-- Doc/library/pprint.rst | 4 ++-- Doc/library/pty.rst | 4 ++-- Doc/library/pwd.rst | 4 ++-- Doc/library/py_compile.rst | 4 ++-- Doc/library/pyclbr.rst | 4 ++-- Doc/library/pydoc.rst | 4 ++-- Doc/library/pyexpat.rst | 4 ++-- Doc/library/queue.rst | 4 ++-- Doc/library/quopri.rst | 4 ++-- Doc/library/random.rst | 4 ++-- Doc/library/re.rst | 4 ++-- Doc/library/readline.rst | 4 ++-- Doc/library/reprlib.rst | 4 ++-- Doc/library/resource.rst | 4 ++-- Doc/library/rlcompleter.rst | 4 ++-- Doc/library/runpy.rst | 4 ++-- Doc/library/sched.rst | 4 ++-- Doc/library/secrets.rst | 4 ++-- Doc/library/select.rst | 4 ++-- Doc/library/selectors.rst | 4 ++-- Doc/library/shelve.rst | 4 ++-- Doc/library/shlex.rst | 4 ++-- Doc/library/shutil.rst | 4 ++-- Doc/library/signal.rst | 4 ++-- Doc/library/site.rst | 4 ++-- Doc/library/smtplib.rst | 4 ++-- Doc/library/socket.rst | 4 ++-- Doc/library/socketserver.rst | 4 ++-- Doc/library/sqlite3.rst | 4 ++-- Doc/library/ssl.rst | 4 ++-- Doc/library/stat.rst | 4 ++-- Doc/library/statistics.rst | 4 ++-- Doc/library/string.rst | 4 ++-- Doc/library/stringprep.rst | 4 ++-- Doc/library/struct.rst | 4 ++-- Doc/library/subprocess.rst | 4 ++-- Doc/library/symtable.rst | 4 ++-- Doc/library/sys.monitoring.rst | 4 ++-- Doc/library/sys.rst | 4 ++-- Doc/library/sysconfig.rst | 4 ++-- Doc/library/syslog.rst | 4 ++-- Doc/library/tabnanny.rst | 4 ++-- Doc/library/tarfile.rst | 4 ++-- Doc/library/tempfile.rst | 4 ++-- Doc/library/termios.rst | 4 ++-- Doc/library/test.rst | 4 ++-- Doc/library/textwrap.rst | 4 ++-- Doc/library/threading.rst | 4 ++-- Doc/library/time.rst | 4 ++-- Doc/library/timeit.rst | 4 ++-- Doc/library/tkinter.colorchooser.rst | 4 ++-- Doc/library/tkinter.dnd.rst | 4 ++-- Doc/library/tkinter.font.rst | 4 ++-- Doc/library/tkinter.messagebox.rst | 4 ++-- Doc/library/tkinter.rst | 4 ++-- Doc/library/tkinter.scrolledtext.rst | 4 ++-- Doc/library/tkinter.ttk.rst | 4 ++-- Doc/library/token.rst | 4 ++-- Doc/library/tokenize.rst | 4 ++-- Doc/library/tomllib.rst | 4 ++-- Doc/library/trace.rst | 4 ++-- Doc/library/traceback.rst | 4 ++-- Doc/library/tracemalloc.rst | 4 ++-- Doc/library/tty.rst | 4 ++-- Doc/library/types.rst | 4 ++-- Doc/library/unicodedata.rst | 4 ++-- Doc/library/unittest.mock-examples.rst | 4 ++-- Doc/library/unittest.mock.rst | 5 ++--- Doc/library/unittest.rst | 4 ++-- Doc/library/urllib.error.rst | 4 ++-- Doc/library/urllib.parse.rst | 4 ++-- Doc/library/urllib.request.rst | 4 ++-- Doc/library/urllib.robotparser.rst | 4 ++-- Doc/library/urllib.rst | 4 ++-- Doc/library/uuid.rst | 4 ++-- Doc/library/venv.rst | 4 ++-- Doc/library/warnings.rst | 4 ++-- Doc/library/wave.rst | 4 ++-- Doc/library/webbrowser.rst | 4 ++-- Doc/library/winreg.rst | 4 ++-- Doc/library/winsound.rst | 4 ++-- Doc/library/wsgiref.rst | 4 ++-- Doc/library/xml.dom.minidom.rst | 4 ++-- Doc/library/xml.dom.pulldom.rst | 4 ++-- Doc/library/xml.dom.rst | 4 ++-- Doc/library/xml.etree.elementtree.rst | 4 ++-- Doc/library/xml.sax.handler.rst | 4 ++-- Doc/library/xml.sax.reader.rst | 4 ++-- Doc/library/xml.sax.rst | 4 ++-- Doc/library/xml.sax.utils.rst | 4 ++-- Doc/library/xmlrpc.client.rst | 4 ++-- Doc/library/xmlrpc.server.rst | 4 ++-- Doc/library/zipapp.rst | 4 ++-- Doc/library/zipfile.rst | 4 ++-- Doc/library/zipimport.rst | 4 ++-- Doc/library/zlib.rst | 4 ++-- Doc/library/zoneinfo.rst | 4 ++-- 220 files changed, 440 insertions(+), 443 deletions(-) diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 762f8b4695b3dd..1ebff4409b1e95 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -1,5 +1,5 @@ -:mod:`__future__` --- Future statement definitions -================================================== +:mod:`!__future__` --- Future statement definitions +=================================================== .. module:: __future__ :synopsis: Future statement definitions diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index c999253f781b10..6232e173d9537d 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -1,5 +1,5 @@ -:mod:`__main__` --- Top-level code environment -============================================== +:mod:`!__main__` --- Top-level code environment +=============================================== .. module:: __main__ :synopsis: The environment where top-level code is run. Covers command-line diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 297f50a46e0692..3842539ce405d9 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -1,5 +1,5 @@ -:mod:`_thread` --- Low-level threading API -========================================== +:mod:`!_thread` --- Low-level threading API +=========================================== .. module:: _thread :synopsis: Low-level threading API. diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 10e2cba50e49b0..168ef3ec00d81b 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -1,5 +1,5 @@ -:mod:`abc` --- Abstract Base Classes -==================================== +:mod:`!abc` --- Abstract Base Classes +===================================== .. module:: abc :synopsis: Abstract base classes according to :pep:`3119`. diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index eaddd44e2defd7..0367c83d9369d3 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1,5 +1,5 @@ -:mod:`argparse` --- Parser for command-line options, arguments and sub-commands -=============================================================================== +:mod:`!argparse` --- Parser for command-line options, arguments and sub-commands +================================================================================ .. module:: argparse :synopsis: Command-line option and argument parsing library. diff --git a/Doc/library/array.rst b/Doc/library/array.rst index cdf21db8779fe8..d34a1888342e27 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -1,5 +1,5 @@ -:mod:`array` --- Efficient arrays of numeric values -=================================================== +:mod:`!array` --- Efficient arrays of numeric values +==================================================== .. module:: array :synopsis: Space efficient arrays of uniformly typed numeric values. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 02dc7c86082502..24c56f17ebb002 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1,5 +1,5 @@ -:mod:`ast` --- Abstract Syntax Trees -==================================== +:mod:`!ast` --- Abstract Syntax Trees +===================================== .. module:: ast :synopsis: Abstract Syntax Tree classes and manipulation. diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 5f33c6813e74c0..184f981c1021aa 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -1,5 +1,5 @@ -:mod:`asyncio` --- Asynchronous I/O -=================================== +:mod:`!asyncio` --- Asynchronous I/O +==================================== .. module:: asyncio :synopsis: Asynchronous I/O. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 43a8bd2d7cd133..02d2f0807df8f6 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -1,5 +1,5 @@ -:mod:`atexit` --- Exit handlers -=============================== +:mod:`!atexit` --- Exit handlers +================================ .. module:: atexit :synopsis: Register and execute cleanup functions. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index e596893358f3fb..cec9a6cef4bf7d 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -1,5 +1,5 @@ -:mod:`base64` --- Base16, Base32, Base64, Base85 Data Encodings -=============================================================== +:mod:`!base64` --- Base16, Base32, Base64, Base85 Data Encodings +================================================================ .. module:: base64 :synopsis: RFC 4648: Base16, Base32, Base64 Data Encodings; diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 504fbd0464515b..c873666f325fbc 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -1,5 +1,5 @@ -:mod:`bdb` --- Debugger framework -================================= +:mod:`!bdb` --- Debugger framework +================================== .. module:: bdb :synopsis: Debugger framework. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 39fabb59bb1984..1bab785684bbab 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -1,5 +1,5 @@ -:mod:`binascii` --- Convert between binary and ASCII -==================================================== +:mod:`!binascii` --- Convert between binary and ASCII +===================================================== .. module:: binascii :synopsis: Tools for converting between binary and various ASCII-encoded binary diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 31c79b91061591..78da563397b625 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -1,5 +1,5 @@ -:mod:`bisect` --- Array bisection algorithm -=========================================== +:mod:`!bisect` --- Array bisection algorithm +============================================ .. module:: bisect :synopsis: Array bisection algorithms for binary searching. diff --git a/Doc/library/builtins.rst b/Doc/library/builtins.rst index 7e4f8fe0531567..644344e7fef29a 100644 --- a/Doc/library/builtins.rst +++ b/Doc/library/builtins.rst @@ -1,5 +1,5 @@ -:mod:`builtins` --- Built-in objects -==================================== +:mod:`!builtins` --- Built-in objects +===================================== .. module:: builtins :synopsis: The module that provides the built-in namespace. diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index eaf0a096455fad..ebe2e43febaefa 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -1,5 +1,5 @@ -:mod:`bz2` --- Support for :program:`bzip2` compression -======================================================= +:mod:`!bz2` --- Support for :program:`bzip2` compression +======================================================== .. module:: bz2 :synopsis: Interfaces for bzip2 compression and decompression. diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index e699a7284ac802..d5876054da3eee 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -1,5 +1,5 @@ -:mod:`calendar` --- General calendar-related functions -====================================================== +:mod:`!calendar` --- General calendar-related functions +======================================================= .. module:: calendar :synopsis: Functions for working with calendars, including some emulation diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index fdac51d9603ceb..65e98e09ad7ae3 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -1,5 +1,5 @@ -:mod:`cmath` --- Mathematical functions for complex numbers -=========================================================== +:mod:`!cmath` --- Mathematical functions for complex numbers +============================================================ .. module:: cmath :synopsis: Mathematical functions for complex numbers. diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index 39ef4b481478d1..66544f82f6ff3f 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -1,5 +1,5 @@ -:mod:`cmd` --- Support for line-oriented command interpreters -============================================================= +:mod:`!cmd` --- Support for line-oriented command interpreters +============================================================== .. module:: cmd :synopsis: Build line-oriented command interpreters. diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 8cb604cf48ff0b..c297160e825c1a 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -1,5 +1,5 @@ -:mod:`code` --- Interpreter base classes -======================================== +:mod:`!code` --- Interpreter base classes +========================================= .. module:: code :synopsis: Facilities to implement read-eval-print loops. diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 010ae25557a9c9..2cfd8a1eaee806 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1,5 +1,5 @@ -:mod:`codecs` --- Codec registry and base classes -================================================= +:mod:`!codecs` --- Codec registry and base classes +================================================== .. module:: codecs :synopsis: Encode and decode data and streams. diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 55606e1c5f09ac..16f674adb4b22b 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -1,5 +1,5 @@ -:mod:`codeop` --- Compile Python code -===================================== +:mod:`!codeop` --- Compile Python code +====================================== .. module:: codeop :synopsis: Compile (possibly incomplete) Python code. diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 7bcaba60c6ddbd..ea27436f67f0bf 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -1,5 +1,5 @@ -:mod:`collections.abc` --- Abstract Base Classes for Containers -=============================================================== +:mod:`!collections.abc` --- Abstract Base Classes for Containers +================================================================ .. module:: collections.abc :synopsis: Abstract base classes for containers diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 19b89ad79c7414..2a269712f1814d 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -1,5 +1,5 @@ -:mod:`collections` --- Container datatypes -========================================== +:mod:`!collections` --- Container datatypes +=========================================== .. module:: collections :synopsis: Container datatypes diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index b672a05b39145d..125d62b174088a 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -1,5 +1,5 @@ -:mod:`colorsys` --- Conversions between color systems -===================================================== +:mod:`!colorsys` --- Conversions between color systems +====================================================== .. module:: colorsys :synopsis: Conversion functions between RGB and other color systems. diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index df1eefab839cc1..8375648e6dd2d3 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -1,5 +1,5 @@ -:mod:`compileall` --- Byte-compile Python libraries -=================================================== +:mod:`!compileall` --- Byte-compile Python libraries +==================================================== .. module:: compileall :synopsis: Tools for byte-compiling all Python source files in a directory tree. diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index d3c7a40aa9d390..e3b24451188cc4 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -1,5 +1,5 @@ -:mod:`concurrent.futures` --- Launching parallel tasks -====================================================== +:mod:`!concurrent.futures` --- Launching parallel tasks +======================================================= .. module:: concurrent.futures :synopsis: Execute computations concurrently using threads or processes. diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 9e7638d087a7ce..e84fb513e45267 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -1,5 +1,5 @@ -:mod:`configparser` --- Configuration file parser -================================================= +:mod:`!configparser` --- Configuration file parser +================================================== .. module:: configparser :synopsis: Configuration file parser. diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 647832447de946..8ae386b489fb4e 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -1,5 +1,5 @@ -:mod:`contextvars` --- Context Variables -======================================== +:mod:`!contextvars` --- Context Variables +========================================= .. module:: contextvars :synopsis: Context Variables diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 74333b2e934814..95b41f988a035b 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -1,5 +1,5 @@ -:mod:`copy` --- Shallow and deep copy operations -================================================ +:mod:`!copy` --- Shallow and deep copy operations +================================================= .. module:: copy :synopsis: Shallow and deep copy operations. diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 2a28c043f80723..6e3144824ebe91 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -1,5 +1,5 @@ -:mod:`copyreg` --- Register :mod:`pickle` support functions -=========================================================== +:mod:`!copyreg` --- Register :mod:`!pickle` support functions +============================================================= .. module:: copyreg :synopsis: Register pickle support functions. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index d17468023c6de1..3ed199750affab 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -1,5 +1,5 @@ -:mod:`csv` --- CSV File Reading and Writing -=========================================== +:mod:`!csv` --- CSV File Reading and Writing +============================================ .. module:: csv :synopsis: Write and read tabular data to and from delimited files. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 1deaa3a0f3899f..820535e3cba106 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1,5 +1,5 @@ -:mod:`ctypes` --- A foreign function library for Python -======================================================= +:mod:`!ctypes` --- A foreign function library for Python +======================================================== .. module:: ctypes :synopsis: A foreign function library for Python. diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index 410b76e77c025b..cb895664ff1b11 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -1,5 +1,5 @@ -:mod:`curses.ascii` --- Utilities for ASCII characters -====================================================== +:mod:`!curses.ascii` --- Utilities for ASCII characters +======================================================= .. module:: curses.ascii :synopsis: Constants and set-membership functions for ASCII characters. diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index d770c03c8375f4..11fd841d381f69 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -1,5 +1,5 @@ -:mod:`curses.panel` --- A panel stack extension for curses -========================================================== +:mod:`!curses.panel` --- A panel stack extension for curses +=========================================================== .. module:: curses.panel :synopsis: A panel stack extension that adds depth to curses windows. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 550872ce2ca59e..883150e91378cc 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1,5 +1,5 @@ -:mod:`curses` --- Terminal handling for character-cell displays -=============================================================== +:mod:`!curses` --- Terminal handling for character-cell displays +================================================================ .. module:: curses :synopsis: An interface to the curses library, providing portable diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index b20b2027cfcc93..0723d0fe2fceb0 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1,5 +1,5 @@ -:mod:`datetime` --- Basic date and time types -============================================= +:mod:`!datetime` --- Basic date and time types +============================================== .. module:: datetime :synopsis: Basic date and time types. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 54627363ba76ae..77148a558d1909 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -1,5 +1,5 @@ -:mod:`dbm` --- Interfaces to Unix "databases" -============================================= +:mod:`!dbm` --- Interfaces to Unix "databases" +============================================== .. module:: dbm :synopsis: Interfaces to various Unix "database" formats. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 3c51dbc04dc92e..3e33581f96f16a 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1,5 +1,5 @@ -:mod:`decimal` --- Decimal fixed point and floating point arithmetic -==================================================================== +:mod:`!decimal` --- Decimal fixed point and floating point arithmetic +===================================================================== .. module:: decimal :synopsis: Implementation of the General Decimal Arithmetic Specification. diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 6a34f2e4d7f5e1..ce948a6860f02c 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -1,5 +1,5 @@ -:mod:`difflib` --- Helpers for computing deltas -=============================================== +:mod:`!difflib` --- Helpers for computing deltas +================================================ .. module:: difflib :synopsis: Helpers for computing differences between objects. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index e255fad55e4a25..b9e2efab827384 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1,5 +1,5 @@ -:mod:`dis` --- Disassembler for Python bytecode -=============================================== +:mod:`!dis` --- Disassembler for Python bytecode +================================================ .. module:: dis :synopsis: Disassembler for Python bytecode. diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 5f7d10a6dce037..6b0282eed49566 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1,5 +1,5 @@ -:mod:`doctest` --- Test interactive Python examples -=================================================== +:mod:`!doctest` --- Test interactive Python examples +==================================================== .. module:: doctest :synopsis: Test pieces of code within docstrings. diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index aa0134412f3a60..6875af2be49d7a 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -1,5 +1,5 @@ -:mod:`email.charset`: Representing character sets -------------------------------------------------- +:mod:`!email.charset`: Representing character sets +-------------------------------------------------- .. module:: email.charset :synopsis: Character Sets diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 5b49339650f0e9..34121f8c0a7727 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -1,5 +1,5 @@ -:mod:`email.contentmanager`: Managing MIME Content --------------------------------------------------- +:mod:`!email.contentmanager`: Managing MIME Content +--------------------------------------------------- .. module:: email.contentmanager :synopsis: Storing and Retrieving Content from MIME Parts diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 3bd377e33f6c15..9c8c8c9234ed7a 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -1,5 +1,5 @@ -:mod:`email.encoders`: Encoders -------------------------------- +:mod:`!email.encoders`: Encoders +-------------------------------- .. module:: email.encoders :synopsis: Encoders for email message payloads. diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 56aea6598b8615..33ab4265116178 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -1,5 +1,5 @@ -:mod:`email.errors`: Exception and Defect classes -------------------------------------------------- +:mod:`!email.errors`: Exception and Defect classes +-------------------------------------------------- .. module:: email.errors :synopsis: The exception classes used by the email package. diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index afa0038ea2d6c4..a3132d02687bc9 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -1,5 +1,5 @@ -:mod:`email.generator`: Generating MIME documents -------------------------------------------------- +:mod:`!email.generator`: Generating MIME documents +-------------------------------------------------- .. module:: email.generator :synopsis: Generate flat text email messages from a message structure. diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index e093f138936b36..6e230d5faf1654 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -1,5 +1,5 @@ -:mod:`email.header`: Internationalized headers ----------------------------------------------- +:mod:`!email.header`: Internationalized headers +----------------------------------------------- .. module:: email.header :synopsis: Representing non-ASCII headers diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 00a954e0307ea6..bcbd00c833e28e 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -1,5 +1,5 @@ -:mod:`email.headerregistry`: Custom Header Objects --------------------------------------------------- +:mod:`!email.headerregistry`: Custom Header Objects +--------------------------------------------------- .. module:: email.headerregistry :synopsis: Automatic Parsing of headers based on the field name diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index d53ab33b8904a7..090981d84b4de3 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -1,5 +1,5 @@ -:mod:`email.iterators`: Iterators ---------------------------------- +:mod:`!email.iterators`: Iterators +---------------------------------- .. module:: email.iterators :synopsis: Iterate over a message object tree. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index c10e1b50d4903b..e9cce1af186526 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -1,5 +1,5 @@ -:mod:`email.message`: Representing an email message ---------------------------------------------------- +:mod:`!email.message`: Representing an email message +---------------------------------------------------- .. module:: email.message :synopsis: The base class representing email messages. diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index dc0dd3b9eebde6..b85673a4acd0d0 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -1,5 +1,5 @@ -:mod:`email.mime`: Creating email and MIME objects from scratch ---------------------------------------------------------------- +:mod:`!email.mime`: Creating email and MIME objects from scratch +---------------------------------------------------------------- .. module:: email.mime :synopsis: Build MIME messages. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index dda0466a6afa7d..439b5c8f34b65a 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -1,5 +1,5 @@ -:mod:`email.parser`: Parsing email messages -------------------------------------------- +:mod:`!email.parser`: Parsing email messages +-------------------------------------------- .. module:: email.parser :synopsis: Parse flat text email messages to produce a message object structure. diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index f4777bb2462138..83feedf728351e 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -1,5 +1,5 @@ -:mod:`email.policy`: Policy Objects ------------------------------------ +:mod:`!email.policy`: Policy Objects +------------------------------------ .. module:: email.policy :synopsis: Controlling the parsing and generating of messages diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 3a6039004fcaae..66c42e4a5008ee 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -1,5 +1,5 @@ -:mod:`email` --- An email and MIME handling package -=================================================== +:mod:`!email` --- An email and MIME handling package +==================================================== .. module:: email :synopsis: Package supporting the parsing, manipulating, and generating diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index d693a9bc3933b5..6f0bed130bc64c 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -1,5 +1,5 @@ -:mod:`email.utils`: Miscellaneous utilities -------------------------------------------- +:mod:`!email.utils`: Miscellaneous utilities +-------------------------------------------- .. module:: email.utils :synopsis: Miscellaneous email package utilities. diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index 168e45cfd6fc90..518a2940edcf69 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -1,5 +1,5 @@ -:mod:`ensurepip` --- Bootstrapping the ``pip`` installer -======================================================== +:mod:`!ensurepip` --- Bootstrapping the ``pip`` installer +========================================================= .. module:: ensurepip :synopsis: Bootstrapping the "pip" installer into an existing Python diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 21f41b73086c98..8c604c2347a547 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -1,5 +1,5 @@ -:mod:`enum` --- Support for enumerations -======================================== +:mod:`!enum` --- Support for enumerations +========================================= .. module:: enum :synopsis: Implementation of an enumeration class. diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 283e8b013265d9..4983b8961b1c3f 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -1,5 +1,5 @@ -:mod:`errno` --- Standard errno system symbols -============================================== +:mod:`!errno` --- Standard errno system symbols +=============================================== .. module:: errno :synopsis: Standard errno system symbols. diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index c40d5e9aacb83c..4067d7912b88b2 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -1,5 +1,5 @@ -:mod:`faulthandler` --- Dump the Python traceback -================================================= +:mod:`!faulthandler` --- Dump the Python traceback +================================================== .. module:: faulthandler :synopsis: Dump the Python traceback. diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 59215f34e01cb7..7bd64e43dd5bfe 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -1,5 +1,5 @@ -:mod:`fcntl` --- The ``fcntl`` and ``ioctl`` system calls -========================================================= +:mod:`!fcntl` --- The ``fcntl`` and ``ioctl`` system calls +========================================================== .. module:: fcntl :platform: Unix diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index 42d20b9c201783..2a0670ffcc2cbc 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -1,5 +1,5 @@ -:mod:`filecmp` --- File and Directory Comparisons -================================================= +:mod:`!filecmp` --- File and Directory Comparisons +================================================== .. module:: filecmp :synopsis: Compare files efficiently. diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index f93e9a58791eeb..94a4139f64c2e4 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -1,5 +1,5 @@ -:mod:`fileinput` --- Iterate over lines from multiple input streams -=================================================================== +:mod:`!fileinput` --- Iterate over lines from multiple input streams +==================================================================== .. module:: fileinput :synopsis: Loop over standard input or a list of files. diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 7cddecd5e80887..fda44923f204fc 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -1,5 +1,5 @@ -:mod:`fnmatch` --- Unix filename pattern matching -================================================= +:mod:`!fnmatch` --- Unix filename pattern matching +================================================== .. module:: fnmatch :synopsis: Unix shell style filename pattern matching. diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 887c3844d20faa..552d6030b1ceda 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -1,5 +1,5 @@ -:mod:`fractions` --- Rational numbers -===================================== +:mod:`!fractions` --- Rational numbers +====================================== .. module:: fractions :synopsis: Rational numbers. diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 8d1aae018ada12..8c39dc00f5db02 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -1,5 +1,5 @@ -:mod:`ftplib` --- FTP protocol client -===================================== +:mod:`!ftplib` --- FTP protocol client +====================================== .. module:: ftplib :synopsis: FTP protocol client (requires sockets). diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 82c970d25a7aac..f3f2f1169f7986 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -1,5 +1,5 @@ -:mod:`functools` --- Higher-order functions and operations on callable objects -============================================================================== +:mod:`!functools` --- Higher-order functions and operations on callable objects +=============================================================================== .. module:: functools :synopsis: Higher-order functions and operations on callable objects. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index e36a71af2b64ab..9e3f942904dc54 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -1,5 +1,5 @@ -:mod:`gc` --- Garbage Collector interface -========================================= +:mod:`!gc` --- Garbage Collector interface +========================================== .. module:: gc :synopsis: Interface to the cycle-detecting garbage collector. diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index ada68b240143e8..d43d3250732306 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -1,5 +1,5 @@ -:mod:`getopt` --- C-style parser for command line options -========================================================= +:mod:`!getopt` --- C-style parser for command line options +========================================================== .. module:: getopt :synopsis: Portable parser for command line options; support both short and diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index 54c84d45a59856..9d67250033df81 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -1,5 +1,5 @@ -:mod:`getpass` --- Portable password input -========================================== +:mod:`!getpass` --- Portable password input +=========================================== .. module:: getpass :synopsis: Portable reading of passwords and retrieval of the userid. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 41beac3e0c7396..d0de83907eb297 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -1,5 +1,5 @@ -:mod:`gettext` --- Multilingual internationalization services -============================================================= +:mod:`!gettext` --- Multilingual internationalization services +============================================================== .. module:: gettext :synopsis: Multilingual internationalization services. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index ab6da98bc74ad2..684466d354aef8 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -1,5 +1,5 @@ -:mod:`glob` --- Unix style pathname pattern expansion -===================================================== +:mod:`!glob` --- Unix style pathname pattern expansion +====================================================== .. module:: glob :synopsis: Unix shell style pathname pattern expansion. diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 5414d6370b78ce..a0b16576fad219 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -1,5 +1,5 @@ -:mod:`graphlib` --- Functionality to operate with graph-like structures -========================================================================= +:mod:`!graphlib` --- Functionality to operate with graph-like structures +======================================================================== .. module:: graphlib :synopsis: Functionality to operate with graph-like structures diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index 9cf25b7ae137a3..30caea328baa79 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -1,5 +1,5 @@ -:mod:`grp` --- The group database -================================= +:mod:`!grp` --- The group database +================================== .. module:: grp :platform: Unix diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index e6106e72a83d25..c609d54b73883e 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -1,5 +1,5 @@ -:mod:`gzip` --- Support for :program:`gzip` files -================================================= +:mod:`!gzip` --- Support for :program:`gzip` files +================================================== .. module:: gzip :synopsis: Interfaces for gzip compression and decompression using file objects. diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index aa0c6fc503e8ff..8cf413fbd1e005 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -1,5 +1,5 @@ -:mod:`hashlib` --- Secure hashes and message digests -==================================================== +:mod:`!hashlib` --- Secure hashes and message digests +===================================================== .. module:: hashlib :synopsis: Secure hash and message digest algorithms. diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index ad407141a2f590..d3c4b920ba500a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -1,5 +1,5 @@ -:mod:`heapq` --- Heap queue algorithm -===================================== +:mod:`!heapq` --- Heap queue algorithm +====================================== .. module:: heapq :synopsis: Heap queue algorithm (a.k.a. priority queue). diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index 43012e03c580e8..d6692033b2d4c3 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -1,5 +1,5 @@ -:mod:`hmac` --- Keyed-Hashing for Message Authentication -======================================================== +:mod:`!hmac` --- Keyed-Hashing for Message Authentication +========================================================= .. module:: hmac :synopsis: Keyed-Hashing for Message Authentication (HMAC) implementation diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index 10529561a92cd0..add18e4c87d220 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -1,5 +1,5 @@ -:mod:`html.entities` --- Definitions of HTML general entities -============================================================= +:mod:`!html.entities` --- Definitions of HTML general entities +============================================================== .. module:: html.entities :synopsis: Definitions of HTML general entities. diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index d35090111e0822..6d433b5a04fc4a 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -1,5 +1,5 @@ -:mod:`html.parser` --- Simple HTML and XHTML parser -=================================================== +:mod:`!html.parser` --- Simple HTML and XHTML parser +==================================================== .. module:: html.parser :synopsis: A simple parser that can handle HTML and XHTML. diff --git a/Doc/library/html.rst b/Doc/library/html.rst index c2b01e14ea7555..9aa39ba9a42b0f 100644 --- a/Doc/library/html.rst +++ b/Doc/library/html.rst @@ -1,5 +1,5 @@ -:mod:`html` --- HyperText Markup Language support -================================================= +:mod:`!html` --- HyperText Markup Language support +================================================== .. module:: html :synopsis: Helpers for manipulating HTML. diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 7e4502064f22a1..2835c8d0eb711e 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -1,5 +1,5 @@ -:mod:`http.client` --- HTTP protocol client -=========================================== +:mod:`!http.client` --- HTTP protocol client +============================================ .. module:: http.client :synopsis: HTTP and HTTPS protocol client (requires sockets). diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 2fe188be641c2d..31ac8bafb6ab4b 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -1,5 +1,5 @@ -:mod:`http.cookiejar` --- Cookie handling for HTTP clients -========================================================== +:mod:`!http.cookiejar` --- Cookie handling for HTTP clients +=========================================================== .. module:: http.cookiejar :synopsis: Classes for automatic handling of HTTP cookies. diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index e91972fe621a48..4ce2e3c4f4cb42 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -1,5 +1,5 @@ -:mod:`http.cookies` --- HTTP state management -============================================= +:mod:`!http.cookies` --- HTTP state management +============================================== .. module:: http.cookies :synopsis: Support for HTTP state management (cookies). diff --git a/Doc/library/http.rst b/Doc/library/http.rst index 998d6e73f9dd82..ce3fb9f8120502 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -1,5 +1,5 @@ -:mod:`http` --- HTTP modules -============================ +:mod:`!http` --- HTTP modules +============================= .. module:: http :synopsis: HTTP status codes and messages diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 886e359bd8cd62..3c80fa747d5f1f 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -1,5 +1,5 @@ -:mod:`http.server` --- HTTP servers -=================================== +:mod:`!http.server` --- HTTP servers +==================================== .. module:: http.server :synopsis: HTTP server and request handlers. diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index ccfd0cd3dde109..2418a3b6b719e7 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -1,5 +1,5 @@ -:mod:`imaplib` --- IMAP4 protocol client -======================================== +:mod:`!imaplib` --- IMAP4 protocol client +========================================= .. module:: imaplib :synopsis: IMAP4 protocol client (requires sockets). diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index c25c48530722e6..7261eacea905f0 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources.abc` -- Abstract base classes for resources ---------------------------------------------------------------------- +:mod:`!importlib.resources.abc` -- Abstract base classes for resources +---------------------------------------------------------------------- .. module:: importlib.resources.abc :synopsis: Abstract base classes for resources diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 9a5e4c76e7bd8f..e002198899c8b8 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources` -- Package resource reading, opening and access --------------------------------------------------------------------------- +:mod:`!importlib.resources` -- Package resource reading, opening and access +--------------------------------------------------------------------------- .. module:: importlib.resources :synopsis: Package resource reading, opening, and access diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 4a0a090facb8bb..7130faa4b5b696 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1,5 +1,5 @@ -:mod:`inspect` --- Inspect live objects -======================================= +:mod:`!inspect` --- Inspect live objects +======================================== .. testsetup:: * diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 8eb531aa4ea248..748c49968f505c 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -1,5 +1,5 @@ -:mod:`io` --- Core tools for working with streams -================================================= +:mod:`!io` --- Core tools for working with streams +================================================== .. module:: io :synopsis: Core tools for working with streams. diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index a4073a4dac86b9..d7dccf1a86593d 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -1,5 +1,5 @@ -:mod:`ipaddress` --- IPv4/IPv6 manipulation library -=================================================== +:mod:`!ipaddress` --- IPv4/IPv6 manipulation library +==================================================== .. module:: ipaddress :synopsis: IPv4/IPv6 manipulation library. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index fb3f33370e3faa..9a1ed3594e81e3 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1,5 +1,5 @@ -:mod:`itertools` --- Functions creating iterators for efficient looping -======================================================================= +:mod:`!itertools` --- Functions creating iterators for efficient looping +======================================================================== .. module:: itertools :synopsis: Functions creating iterators for efficient looping. diff --git a/Doc/library/json.rst b/Doc/library/json.rst index c82ff9dc325b4c..42cb1f850fe9c5 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -1,5 +1,5 @@ -:mod:`json` --- JSON encoder and decoder -======================================== +:mod:`!json` --- JSON encoder and decoder +========================================= .. module:: json :synopsis: Encode and decode the JSON format. diff --git a/Doc/library/keyword.rst b/Doc/library/keyword.rst index c3b4699cb05af6..ac57140f888024 100644 --- a/Doc/library/keyword.rst +++ b/Doc/library/keyword.rst @@ -1,5 +1,5 @@ -:mod:`keyword` --- Testing for Python keywords -============================================== +:mod:`!keyword` --- Testing for Python keywords +=============================================== .. module:: keyword :synopsis: Test whether a string is a keyword in Python. diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index dd9f4ee45ba82e..88c6079a05b7fa 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -1,5 +1,5 @@ -:mod:`linecache` --- Random access to text lines -================================================ +:mod:`!linecache` --- Random access to text lines +================================================= .. module:: linecache :synopsis: Provides random access to individual lines from text files. diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 414979524e57b6..0a8cbd4f95f473 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -1,5 +1,5 @@ -:mod:`locale` --- Internationalization services -=============================================== +:mod:`!locale` --- Internationalization services +================================================ .. module:: locale :synopsis: Internationalization services. diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 13850c91446da5..dfbf0b1cf2f9ff 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -1,5 +1,5 @@ -:mod:`logging.config` --- Logging configuration -=============================================== +:mod:`!logging.config` --- Logging configuration +================================================ .. module:: logging.config :synopsis: Configuration of the logging module. diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 2fe9370333beaf..68a54c2ead25bc 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -1,5 +1,5 @@ -:mod:`logging.handlers` --- Logging handlers -============================================ +:mod:`!logging.handlers` --- Logging handlers +============================================= .. module:: logging.handlers :synopsis: Handlers for the logging module. diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index a733b288ecb6d0..fb6ca38ba72aba 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1,5 +1,5 @@ -:mod:`logging` --- Logging facility for Python -============================================== +:mod:`!logging` --- Logging facility for Python +=============================================== .. module:: logging :synopsis: Flexible event logging system for applications. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 74bf2670f9def6..69f7cb8d48d7ae 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -1,5 +1,5 @@ -:mod:`lzma` --- Compression using the LZMA algorithm -==================================================== +:mod:`!lzma` --- Compression using the LZMA algorithm +===================================================== .. module:: lzma :synopsis: A Python wrapper for the liblzma compression library. diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index a613548c9e518e..40ea71cd342b47 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -1,5 +1,5 @@ -:mod:`mailbox` --- Manipulate mailboxes in various formats -========================================================== +:mod:`!mailbox` --- Manipulate mailboxes in various formats +=========================================================== .. module:: mailbox :synopsis: Manipulate mailboxes in various formats diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index c6a006b7b4028a..c8f1d57317ea68 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -1,5 +1,5 @@ -:mod:`marshal` --- Internal Python object serialization -======================================================= +:mod:`!marshal` --- Internal Python object serialization +======================================================== .. module:: marshal :synopsis: Convert Python objects to streams of bytes and back (with different diff --git a/Doc/library/math.rst b/Doc/library/math.rst index ab9236cc7cfb81..dc43874bf4eabd 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -1,5 +1,5 @@ -:mod:`math` --- Mathematical functions -====================================== +:mod:`!math` --- Mathematical functions +======================================= .. module:: math :synopsis: Mathematical functions (sin() etc.). diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index a24eab21d57343..91e8c30f8607b3 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -1,5 +1,5 @@ -:mod:`mimetypes` --- Map filenames to MIME types -================================================ +:mod:`!mimetypes` --- Map filenames to MIME types +================================================= .. module:: mimetypes :synopsis: Mapping of filename extensions to MIME types. diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 758721433f77de..4e20c07331a220 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -1,5 +1,5 @@ -:mod:`mmap` --- Memory-mapped file support -========================================== +:mod:`!mmap` --- Memory-mapped file support +=========================================== .. module:: mmap :synopsis: Interface to memory-mapped files for Unix and Windows. diff --git a/Doc/library/modulefinder.rst b/Doc/library/modulefinder.rst index 526f0ff868c2b7..823d853f1ed8eb 100644 --- a/Doc/library/modulefinder.rst +++ b/Doc/library/modulefinder.rst @@ -1,5 +1,5 @@ -:mod:`modulefinder` --- Find modules used by a script -===================================================== +:mod:`!modulefinder` --- Find modules used by a script +====================================================== .. module:: modulefinder :synopsis: Find modules used by a script. diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 72da777cd1407e..327cc3602b1a77 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -1,5 +1,5 @@ -:mod:`msvcrt` --- Useful routines from the MS VC++ runtime -========================================================== +:mod:`!msvcrt` --- Useful routines from the MS VC++ runtime +=========================================================== .. module:: msvcrt :platform: Windows diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index afc148c78e97bd..ff42dcf5f5b299 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1,5 +1,5 @@ -:mod:`multiprocessing` --- Process-based parallelism -==================================================== +:mod:`!multiprocessing` --- Process-based parallelism +===================================================== .. module:: multiprocessing :synopsis: Process-based parallelism. diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 933fd07d62418a..e8f04a6ac7b95d 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -1,5 +1,5 @@ -:mod:`multiprocessing.shared_memory` --- Shared memory for direct access across processes -========================================================================================= +:mod:`!multiprocessing.shared_memory` --- Shared memory for direct access across processes +========================================================================================== .. module:: multiprocessing.shared_memory :synopsis: Provides shared memory for direct access across processes. diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index c36e5cfecfc6a8..f6260383b2b057 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -1,6 +1,5 @@ - -:mod:`netrc` --- netrc file processing -====================================== +:mod:`!netrc` --- netrc file processing +======================================= .. module:: netrc :synopsis: Loading of .netrc files. diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 306bdd94aaca13..5f59746fa59812 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -1,5 +1,5 @@ -:mod:`numbers` --- Numeric abstract base classes -================================================ +:mod:`!numbers` --- Numeric abstract base classes +================================================= .. module:: numbers :synopsis: Numeric abstract base classes (Complex, Real, Integral, etc.). diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 96f2c287875d41..a9a6026af406fe 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -1,5 +1,5 @@ -:mod:`operator` --- Standard operators as functions -=================================================== +:mod:`!operator` --- Standard operators as functions +==================================================== .. module:: operator :synopsis: Functions corresponding to the standard operators. diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 015e83ed2ce5f7..8c7d77d369b44c 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -1,5 +1,5 @@ -:mod:`optparse` --- Parser for command line options -=================================================== +:mod:`!optparse` --- Parser for command line options +==================================================== .. module:: optparse :synopsis: Command-line option parsing library. diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 24d423eb5fbfed..b582321515db56 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -1,5 +1,5 @@ -:mod:`os.path` --- Common pathname manipulations -================================================ +:mod:`!os.path` --- Common pathname manipulations +================================================= .. module:: os.path :synopsis: Operations on pathnames. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 6c92eed9c063f1..eb558357705ad6 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1,5 +1,5 @@ -:mod:`os` --- Miscellaneous operating system interfaces -======================================================= +:mod:`!os` --- Miscellaneous operating system interfaces +======================================================== .. module:: os :synopsis: Miscellaneous operating system interfaces. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 2e18e41869376e..5bf021fbc43210 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1,6 +1,5 @@ - -:mod:`pathlib` --- Object-oriented filesystem paths -=================================================== +:mod:`!pathlib` --- Object-oriented filesystem paths +==================================================== .. module:: pathlib :synopsis: Object-oriented filesystem paths diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 223c27237e4d34..fe589cf5cc8950 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -1,5 +1,5 @@ -:mod:`pickle` --- Python object serialization -============================================= +:mod:`!pickle` --- Python object serialization +============================================== .. module:: pickle :synopsis: Convert Python objects to streams of bytes and back. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 9739207a224431..e072605974f6c2 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -1,5 +1,5 @@ -:mod:`pickletools` --- Tools for pickle developers -================================================== +:mod:`!pickletools` --- Tools for pickle developers +=================================================== .. module:: pickletools :synopsis: Contains extensive comments about the pickle protocols and diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 891a867d1ceb68..5d4ff34ba029a0 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -1,5 +1,5 @@ -:mod:`pkgutil` --- Package extension utility -============================================ +:mod:`!pkgutil` --- Package extension utility +============================================= .. module:: pkgutil :synopsis: Utilities for the import system. diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 66af37e3073852..f082393ef9363c 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -1,5 +1,5 @@ -:mod:`platform` --- Access to underlying platform's identifying data -===================================================================== +:mod:`!platform` --- Access to underlying platform's identifying data +====================================================================== .. module:: platform :synopsis: Retrieves as much platform identifying data as possible. diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 7416ca2650bab4..78b3c2697bd696 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -1,5 +1,5 @@ -:mod:`plistlib` --- Generate and parse Apple ``.plist`` files -============================================================= +:mod:`!plistlib` --- Generate and parse Apple ``.plist`` files +============================================================== .. module:: plistlib :synopsis: Generate and parse Apple plist files. diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index 943eb21f6eec02..23f20b00e6dc6d 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -1,5 +1,5 @@ -:mod:`poplib` --- POP3 protocol client -====================================== +:mod:`!poplib` --- POP3 protocol client +======================================= .. module:: poplib :synopsis: POP3 protocol client (requires sockets). diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 5871574b442667..14ab3e91e8a8e4 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -1,5 +1,5 @@ -:mod:`posix` --- The most common POSIX system calls -=================================================== +:mod:`!posix` --- The most common POSIX system calls +==================================================== .. module:: posix :platform: Unix diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 6dfea25d755f75..df706c10ce9ec4 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -1,5 +1,5 @@ -:mod:`pprint` --- Data pretty printer -===================================== +:mod:`!pprint` --- Data pretty printer +====================================== .. module:: pprint :synopsis: Data pretty printer. diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index bd2f5ed45cb8b4..1a44bb13a841de 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -1,5 +1,5 @@ -:mod:`pty` --- Pseudo-terminal utilities -======================================== +:mod:`!pty` --- Pseudo-terminal utilities +========================================= .. module:: pty :platform: Unix diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index a6c6d79b60b20a..e1ff32912132f7 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -1,5 +1,5 @@ -:mod:`pwd` --- The password database -==================================== +:mod:`!pwd` --- The password database +===================================== .. module:: pwd :platform: Unix diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index a35fa0ba3f7bde..75aa739d1003b8 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -1,5 +1,5 @@ -:mod:`py_compile` --- Compile Python source files -================================================= +:mod:`!py_compile` --- Compile Python source files +================================================== .. module:: py_compile :synopsis: Generate byte-code files from Python source files. diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 1e9876849b02f3..da359004e6dd1f 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -1,5 +1,5 @@ -:mod:`pyclbr` --- Python module browser support -=============================================== +:mod:`!pyclbr` --- Python module browser support +================================================ .. module:: pyclbr :synopsis: Supports information extraction for a Python module browser. diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index df969b2fc7c04c..f7ca1e045699eb 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -1,5 +1,5 @@ -:mod:`pydoc` --- Documentation generator and online help system -=============================================================== +:mod:`!pydoc` --- Documentation generator and online help system +================================================================ .. module:: pydoc :synopsis: Documentation generator and online help system. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 2c048b8290d014..c0e9999f4b1270 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -1,5 +1,5 @@ -:mod:`xml.parsers.expat` --- Fast XML parsing using Expat -========================================================= +:mod:`!xml.parsers.expat` --- Fast XML parsing using Expat +========================================================== .. module:: xml.parsers.expat :synopsis: An interface to the Expat non-validating XML parser. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index fce23313c7de28..fbbebcf4ed8f92 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -1,5 +1,5 @@ -:mod:`queue` --- A synchronized queue class -=========================================== +:mod:`!queue` --- A synchronized queue class +============================================ .. module:: queue :synopsis: A synchronized queue class. diff --git a/Doc/library/quopri.rst b/Doc/library/quopri.rst index 86717c00c3c136..977cb08d836afe 100644 --- a/Doc/library/quopri.rst +++ b/Doc/library/quopri.rst @@ -1,5 +1,5 @@ -:mod:`quopri` --- Encode and decode MIME quoted-printable data -============================================================== +:mod:`!quopri` --- Encode and decode MIME quoted-printable data +=============================================================== .. module:: quopri :synopsis: Encode and decode files using the MIME quoted-printable encoding. diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 22a799c706b7e2..755d1c8908c966 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -1,5 +1,5 @@ -:mod:`random` --- Generate pseudo-random numbers -================================================ +:mod:`!random` --- Generate pseudo-random numbers +================================================= .. module:: random :synopsis: Generate pseudo-random numbers with various common distributions. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index fe7da856076819..39788de76b558b 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1,5 +1,5 @@ -:mod:`re` --- Regular expression operations -=========================================== +:mod:`!re` --- Regular expression operations +============================================ .. module:: re :synopsis: Regular expression operations. diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 8f8718ec51c41b..5658b93c81dc99 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -1,5 +1,5 @@ -:mod:`readline` --- GNU readline interface -========================================== +:mod:`!readline` --- GNU readline interface +=========================================== .. module:: readline :platform: Unix diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 678a11c6f45490..28c7855dfeeef3 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -1,5 +1,5 @@ -:mod:`reprlib` --- Alternate :func:`repr` implementation -======================================================== +:mod:`!reprlib` --- Alternate :func:`repr` implementation +========================================================= .. module:: reprlib :synopsis: Alternate repr() implementation with size limits. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 4fea8d5cb718c1..dd80b1e6670d92 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -1,5 +1,5 @@ -:mod:`resource` --- Resource usage information -============================================== +:mod:`!resource` --- Resource usage information +=============================================== .. module:: resource :platform: Unix diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst index 8287699c5f013e..91779feb525013 100644 --- a/Doc/library/rlcompleter.rst +++ b/Doc/library/rlcompleter.rst @@ -1,5 +1,5 @@ -:mod:`rlcompleter` --- Completion function for GNU readline -=========================================================== +:mod:`!rlcompleter` --- Completion function for GNU readline +============================================================ .. module:: rlcompleter :synopsis: Python identifier completion, suitable for the GNU readline library. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index f2cb595f495f6b..b07ec6e93f80ab 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -1,5 +1,5 @@ -:mod:`runpy` --- Locating and executing Python modules -====================================================== +:mod:`!runpy` --- Locating and executing Python modules +======================================================= .. module:: runpy :synopsis: Locate and run Python modules without importing them first. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 4c980dd97f9394..517dbe8c321898 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -1,5 +1,5 @@ -:mod:`sched` --- Event scheduler -================================ +:mod:`!sched` --- Event scheduler +================================= .. module:: sched :synopsis: General purpose event scheduler. diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 8f1a68d1d8816c..e609948de552bd 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -1,5 +1,5 @@ -:mod:`secrets` --- Generate secure random numbers for managing secrets -====================================================================== +:mod:`!secrets` --- Generate secure random numbers for managing secrets +======================================================================= .. module:: secrets :synopsis: Generate secure random numbers for managing secrets. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index a0058046d0ce4c..06ebaf0201e0e7 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -1,5 +1,5 @@ -:mod:`select` --- Waiting for I/O completion -============================================ +:mod:`!select` --- Waiting for I/O completion +============================================= .. module:: select :synopsis: Wait for I/O completion on multiple streams. diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index 76cbf91412f763..de8c3ef0ea2275 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -1,5 +1,5 @@ -:mod:`selectors` --- High-level I/O multiplexing -================================================ +:mod:`!selectors` --- High-level I/O multiplexing +================================================= .. module:: selectors :synopsis: High-level I/O multiplexing. diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 1fa614f6584170..6e74a59b82b8ec 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -1,5 +1,5 @@ -:mod:`shelve` --- Python object persistence -=========================================== +:mod:`!shelve` --- Python object persistence +============================================ .. module:: shelve :synopsis: Python object persistence. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index 716420f5e74ffa..a96f0864dc1260 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -1,5 +1,5 @@ -:mod:`shlex` --- Simple lexical analysis -======================================== +:mod:`!shlex` --- Simple lexical analysis +========================================= .. module:: shlex :synopsis: Simple lexical analysis for Unix shell-like languages. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 331acfce3afee3..1a053c32866153 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -1,5 +1,5 @@ -:mod:`shutil` --- High-level file operations -============================================ +:mod:`!shutil` --- High-level file operations +============================================= .. module:: shutil :synopsis: High-level file operations, including copying. diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 05ef45c123b02e..48c6841c648ca4 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -1,5 +1,5 @@ -:mod:`signal` --- Set handlers for asynchronous events -====================================================== +:mod:`!signal` --- Set handlers for asynchronous events +======================================================= .. module:: signal :synopsis: Set handlers for asynchronous events. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index e52bbd32d4d493..1c420419568a90 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -1,5 +1,5 @@ -:mod:`site` --- Site-specific configuration hook -================================================ +:mod:`!site` --- Site-specific configuration hook +================================================= .. module:: site :synopsis: Module responsible for site-specific configuration. diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index aaec2aa1ef1dbe..2511ef7f2ada41 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -1,5 +1,5 @@ -:mod:`smtplib` --- SMTP protocol client -======================================= +:mod:`!smtplib` --- SMTP protocol client +======================================== .. module:: smtplib :synopsis: SMTP protocol client (requires sockets). diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 76af783c6292f9..6405f7f00fcb21 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1,5 +1,5 @@ -:mod:`socket` --- Low-level networking interface -================================================ +:mod:`!socket` --- Low-level networking interface +================================================= .. module:: socket :synopsis: Low-level networking interface. diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 864b1dadb78562..f1f87ea975ca42 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -1,5 +1,5 @@ -:mod:`socketserver` --- A framework for network servers -======================================================= +:mod:`!socketserver` --- A framework for network servers +======================================================== .. module:: socketserver :synopsis: A framework for network servers. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e6961821b639b9..6da8798ddfe0c0 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1,5 +1,5 @@ -:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases -============================================================ +:mod:`!sqlite3` --- DB-API 2.0 interface for SQLite databases +============================================================= .. module:: sqlite3 :synopsis: A DB-API 2.0 implementation using SQLite 3.x. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index a90436286ca819..9c757ce1b8efc4 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1,5 +1,5 @@ -:mod:`ssl` --- TLS/SSL wrapper for socket objects -================================================= +:mod:`!ssl` --- TLS/SSL wrapper for socket objects +================================================== .. module:: ssl :synopsis: TLS/SSL wrapper for socket objects diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index f7a3b7b16fe5c3..8434b2e8c75cf4 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -1,5 +1,5 @@ -:mod:`stat` --- Interpreting :func:`~os.stat` results -===================================================== +:mod:`!stat` --- Interpreting :func:`~os.stat` results +====================================================== .. module:: stat :synopsis: Utilities for interpreting the results of os.stat(), diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index d5a316e45ee3e2..7af8c5ae618dff 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -1,5 +1,5 @@ -:mod:`statistics` --- Mathematical statistics functions -======================================================= +:mod:`!statistics` --- Mathematical statistics functions +======================================================== .. module:: statistics :synopsis: Mathematical statistics functions diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 1867678b2077fc..c3c0d732cf18d4 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -1,5 +1,5 @@ -:mod:`string` --- Common string operations -========================================== +:mod:`!string` --- Common string operations +=========================================== .. module:: string :synopsis: Common string operations. diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index c6d78a356d97bc..37d5adf0fa9541 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -1,5 +1,5 @@ -:mod:`stringprep` --- Internet String Preparation -================================================= +:mod:`!stringprep` --- Internet String Preparation +================================================== .. module:: stringprep :synopsis: String preparation, as per RFC 3453 diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 3e507c1c7e7c85..a2c293443e23d3 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -1,5 +1,5 @@ -:mod:`struct` --- Interpret bytes as packed binary data -======================================================= +:mod:`!struct` --- Interpret bytes as packed binary data +======================================================== .. testsetup:: * diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index bd35fda7d79225..d77dc8b04d01e4 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1,5 +1,5 @@ -:mod:`subprocess` --- Subprocess management -=========================================== +:mod:`!subprocess` --- Subprocess management +============================================ .. module:: subprocess :synopsis: Subprocess management. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 47568387f9a7ce..0480502158433a 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -1,5 +1,5 @@ -:mod:`symtable` --- Access to the compiler's symbol tables -========================================================== +:mod:`!symtable` --- Access to the compiler's symbol tables +=========================================================== .. module:: symtable :synopsis: Interface to the compiler's internal symbol tables. diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 583266daeb56a1..0fa06da522049f 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -1,5 +1,5 @@ -:mod:`sys.monitoring` --- Execution event monitoring -==================================================== +:mod:`!sys.monitoring` --- Execution event monitoring +===================================================== .. module:: sys.monitoring :synopsis: Access and control event monitoring diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 19a396d2300011..e6d39522fc6da3 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1,5 +1,5 @@ -:mod:`sys` --- System-specific parameters and functions -======================================================= +:mod:`!sys` --- System-specific parameters and functions +======================================================== .. module:: sys :synopsis: Access system-specific parameters and functions. diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 2faab212e46eff..9556da808f8c63 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -1,5 +1,5 @@ -:mod:`sysconfig` --- Provide access to Python's configuration information -========================================================================= +:mod:`!sysconfig` --- Provide access to Python's configuration information +========================================================================== .. module:: sysconfig :synopsis: Python's configuration information diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 30bf3f09a24d42..548898a37bc6ea 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -1,5 +1,5 @@ -:mod:`syslog` --- Unix syslog library routines -============================================== +:mod:`!syslog` --- Unix syslog library routines +=============================================== .. module:: syslog :platform: Unix diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index dfe688a2f93e0c..4f61b3dd761400 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -1,5 +1,5 @@ -:mod:`tabnanny` --- Detection of ambiguous indentation -====================================================== +:mod:`!tabnanny` --- Detection of ambiguous indentation +======================================================= .. module:: tabnanny :synopsis: Tool for detecting white space related problems in Python diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index cf21dee83e6e7a..5b624f3533136f 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -1,5 +1,5 @@ -:mod:`tarfile` --- Read and write tar archive files -=================================================== +:mod:`!tarfile` --- Read and write tar archive files +==================================================== .. module:: tarfile :synopsis: Read and write tar-format archive files. diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 9add8500c7788c..f0a81a093b435b 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -1,5 +1,5 @@ -:mod:`tempfile` --- Generate temporary files and directories -============================================================ +:mod:`!tempfile` --- Generate temporary files and directories +============================================================= .. module:: tempfile :synopsis: Generate temporary files and directories. diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index 57705ddc4e6470..0c6f3059fe71d1 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -1,5 +1,5 @@ -:mod:`termios` --- POSIX style tty control -========================================== +:mod:`!termios` --- POSIX style tty control +=========================================== .. module:: termios :platform: Unix diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 92d675b48690ff..0fff1a0ae3b563 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1,5 +1,5 @@ -:mod:`test` --- Regression tests package for Python -=================================================== +:mod:`!test` --- Regression tests package for Python +==================================================== .. module:: test :synopsis: Regression tests package containing the testing suite for Python. diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index 7445410f91808c..deaefeee7b8c99 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -1,5 +1,5 @@ -:mod:`textwrap` --- Text wrapping and filling -============================================= +:mod:`!textwrap` --- Text wrapping and filling +============================================== .. module:: textwrap :synopsis: Text wrapping and filling diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 4cf98a49e11442..747d39aae03a3e 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -1,5 +1,5 @@ -:mod:`threading` --- Thread-based parallelism -============================================= +:mod:`!threading` --- Thread-based parallelism +============================================== .. module:: threading :synopsis: Thread-based parallelism. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index d79ca6e1208107..ef033d59d56185 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -1,5 +1,5 @@ -:mod:`time` --- Time access and conversions -=========================================== +:mod:`!time` --- Time access and conversions +============================================ .. module:: time :synopsis: Time access and conversions. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 616f8365b80f6c..548a3ee0540506 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -1,5 +1,5 @@ -:mod:`timeit` --- Measure execution time of small code snippets -=============================================================== +:mod:`!timeit` --- Measure execution time of small code snippets +================================================================ .. module:: timeit :synopsis: Measure the execution time of small code snippets. diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst index 6e8479c1dea1e2..df2b324fd5d3a7 100644 --- a/Doc/library/tkinter.colorchooser.rst +++ b/Doc/library/tkinter.colorchooser.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.colorchooser` --- Color choosing dialog -===================================================== +:mod:`!tkinter.colorchooser` --- Color choosing dialog +====================================================== .. module:: tkinter.colorchooser :platform: Tk diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst index 02de0fd331958d..ccf698af228d02 100644 --- a/Doc/library/tkinter.dnd.rst +++ b/Doc/library/tkinter.dnd.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.dnd` --- Drag and drop support -============================================ +:mod:`!tkinter.dnd` --- Drag and drop support +============================================= .. module:: tkinter.dnd :platform: Tk diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index c7c2b7b566cf8f..ed01bd5f483943 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.font` --- Tkinter font wrapper -============================================ +:mod:`!tkinter.font` --- Tkinter font wrapper +============================================= .. module:: tkinter.font :platform: Tk diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst index 56090a0a0e424b..0dc9632ca73304 100644 --- a/Doc/library/tkinter.messagebox.rst +++ b/Doc/library/tkinter.messagebox.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.messagebox` --- Tkinter message prompts -===================================================== +:mod:`!tkinter.messagebox` --- Tkinter message prompts +====================================================== .. module:: tkinter.messagebox :platform: Tk diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 7e5dee1b562df8..f40790c1175800 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -1,5 +1,5 @@ -:mod:`tkinter` --- Python interface to Tcl/Tk -============================================= +:mod:`!tkinter` --- Python interface to Tcl/Tk +============================================== .. module:: tkinter :synopsis: Interface to Tcl/Tk for graphical user interfaces diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index d20365baa38690..763e24929d74b5 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.scrolledtext` --- Scrolled Text Widget -==================================================== +:mod:`!tkinter.scrolledtext` --- Scrolled Text Widget +===================================================== .. module:: tkinter.scrolledtext :platform: Tk diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index bd0d8b3799a0f1..628e9f945ac365 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.ttk` --- Tk themed widgets -======================================== +:mod:`!tkinter.ttk` --- Tk themed widgets +========================================= .. module:: tkinter.ttk :synopsis: Tk themed widget set diff --git a/Doc/library/token.rst b/Doc/library/token.rst index e6dc37d7ad852c..919ff590b72916 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -1,5 +1,5 @@ -:mod:`token` --- Constants used with Python parse trees -======================================================= +:mod:`!token` --- Constants used with Python parse trees +======================================================== .. module:: token :synopsis: Constants representing terminal nodes of the parse tree. diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 92bdb052267a68..f719319a302a23 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -1,5 +1,5 @@ -:mod:`tokenize` --- Tokenizer for Python source -=============================================== +:mod:`!tokenize` --- Tokenizer for Python source +================================================ .. module:: tokenize :synopsis: Lexical scanner for Python source code. diff --git a/Doc/library/tomllib.rst b/Doc/library/tomllib.rst index 406985b84471f2..b523ad93b35f9d 100644 --- a/Doc/library/tomllib.rst +++ b/Doc/library/tomllib.rst @@ -1,5 +1,5 @@ -:mod:`tomllib` --- Parse TOML files -=================================== +:mod:`!tomllib` --- Parse TOML files +==================================== .. module:: tomllib :synopsis: Parse TOML files. diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index 8854905e192b45..cae94ea08e17e5 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -1,5 +1,5 @@ -:mod:`trace` --- Trace or track Python statement execution -========================================================== +:mod:`!trace` --- Trace or track Python statement execution +=========================================================== .. module:: trace :synopsis: Trace or track Python statement execution. diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index ab83e0df10b709..9983b8da427a2a 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -1,5 +1,5 @@ -:mod:`traceback` --- Print or retrieve a stack traceback -======================================================== +:mod:`!traceback` --- Print or retrieve a stack traceback +========================================================= .. module:: traceback :synopsis: Print or retrieve a stack traceback. diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 68432aeaecbcc1..2370d927292eb0 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -1,5 +1,5 @@ -:mod:`tracemalloc` --- Trace memory allocations -=============================================== +:mod:`!tracemalloc` --- Trace memory allocations +================================================ .. module:: tracemalloc :synopsis: Trace memory allocations. diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst index ed63561c40de24..0641d8a68d5e9e 100644 --- a/Doc/library/tty.rst +++ b/Doc/library/tty.rst @@ -1,5 +1,5 @@ -:mod:`tty` --- Terminal control functions -========================================= +:mod:`!tty` --- Terminal control functions +========================================== .. module:: tty :platform: Unix diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 89bc0a600c0af8..116868c24be864 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -1,5 +1,5 @@ -:mod:`types` --- Dynamic type creation and names for built-in types -=================================================================== +:mod:`!types` --- Dynamic type creation and names for built-in types +==================================================================== .. module:: types :synopsis: Names for built-in types. diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 7db47d48022a0e..37dc37513fa15d 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -1,5 +1,5 @@ -:mod:`unicodedata` --- Unicode Database -======================================= +:mod:`!unicodedata` --- Unicode Database +======================================== .. module:: unicodedata :synopsis: Access the Unicode Database. diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index f2bdde80bdbd64..00cc9bfc0a5f2b 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -1,5 +1,5 @@ -:mod:`unittest.mock` --- getting started -======================================== +:mod:`!unittest.mock` --- getting started +========================================= .. moduleauthor:: Michael Foord .. currentmodule:: unittest.mock diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index ee4c7b2ed252b0..9496c35f5267d0 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -1,6 +1,5 @@ - -:mod:`unittest.mock` --- mock object library -============================================ +:mod:`!unittest.mock` --- mock object library +============================================= .. module:: unittest.mock :synopsis: Mock object library. diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 3af29f19c802c7..eb42210e096ecb 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1,5 +1,5 @@ -:mod:`unittest` --- Unit testing framework -========================================== +:mod:`!unittest` --- Unit testing framework +=========================================== .. module:: unittest :synopsis: Unit testing framework for Python. diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst index facb11f42a40c5..1686ddd09caa48 100644 --- a/Doc/library/urllib.error.rst +++ b/Doc/library/urllib.error.rst @@ -1,5 +1,5 @@ -:mod:`urllib.error` --- Exception classes raised by urllib.request -================================================================== +:mod:`!urllib.error` --- Exception classes raised by urllib.request +=================================================================== .. module:: urllib.error :synopsis: Exception classes raised by urllib.request. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 59fb14960ba9f6..cd402e87a8224b 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -1,5 +1,5 @@ -:mod:`urllib.parse` --- Parse URLs into components -================================================== +:mod:`!urllib.parse` --- Parse URLs into components +=================================================== .. module:: urllib.parse :synopsis: Parse URLs into or assemble them from components. diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index c1e60a46774704..bd39f2cbc6570c 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -1,5 +1,5 @@ -:mod:`urllib.request` --- Extensible library for opening URLs -============================================================= +:mod:`!urllib.request` --- Extensible library for opening URLs +============================================================== .. module:: urllib.request :synopsis: Extensible library for opening URLs. diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index b5a49d9c592387..016fcdc75da67a 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -1,5 +1,5 @@ -:mod:`urllib.robotparser` --- Parser for robots.txt -==================================================== +:mod:`!urllib.robotparser` --- Parser for robots.txt +===================================================== .. module:: urllib.robotparser :synopsis: Load a robots.txt file and answer questions about diff --git a/Doc/library/urllib.rst b/Doc/library/urllib.rst index 624e164625556a..7d9f39ef070fc3 100644 --- a/Doc/library/urllib.rst +++ b/Doc/library/urllib.rst @@ -1,5 +1,5 @@ -:mod:`urllib` --- URL handling modules -====================================== +:mod:`!urllib` --- URL handling modules +======================================= .. module:: urllib diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index e2d231da38fd9a..0f2d7820cb25c8 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -1,5 +1,5 @@ -:mod:`uuid` --- UUID objects according to :rfc:`4122` -===================================================== +:mod:`!uuid` --- UUID objects according to :rfc:`4122` +====================================================== .. module:: uuid :synopsis: UUID objects (universally unique identifiers) according to RFC 4122 diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index cdd1fde2e44b00..fff1075c2473eb 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -1,5 +1,5 @@ -:mod:`venv` --- Creation of virtual environments -================================================ +:mod:`!venv` --- Creation of virtual environments +================================================= .. module:: venv :synopsis: Creation of virtual environments. diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 500398636e11ae..c66e65abee426f 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -1,5 +1,5 @@ -:mod:`warnings` --- Warning control -=================================== +:mod:`!warnings` --- Warning control +==================================== .. module:: warnings :synopsis: Issue warning messages and control their disposition. diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index 55b029bc742b24..89664693cc87b4 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -1,5 +1,5 @@ -:mod:`wave` --- Read and write WAV files -======================================== +:mod:`!wave` --- Read and write WAV files +========================================= .. module:: wave :synopsis: Provide an interface to the WAV sound format. diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 3775d9f9245428..334f21f01c14c0 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -1,5 +1,5 @@ -:mod:`webbrowser` --- Convenient web-browser controller -======================================================= +:mod:`!webbrowser` --- Convenient web-browser controller +======================================================== .. module:: webbrowser :synopsis: Easy-to-use controller for web browsers. diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index 06bd4d87eb03c6..b3a824fb69a49f 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -1,5 +1,5 @@ -:mod:`winreg` --- Windows registry access -========================================= +:mod:`!winreg` --- Windows registry access +========================================== .. module:: winreg :platform: Windows diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst index 370c5216652ba7..f7ca9dc57bbe28 100644 --- a/Doc/library/winsound.rst +++ b/Doc/library/winsound.rst @@ -1,5 +1,5 @@ -:mod:`winsound` --- Sound-playing interface for Windows -======================================================= +:mod:`!winsound` --- Sound-playing interface for Windows +======================================================== .. module:: winsound :platform: Windows diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 7fe84a2de1fceb..e46730f1716761 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -1,5 +1,5 @@ -:mod:`wsgiref` --- WSGI Utilities and Reference Implementation -============================================================== +:mod:`!wsgiref` --- WSGI Utilities and Reference Implementation +=============================================================== .. module:: wsgiref :synopsis: WSGI Utilities and Reference Implementation. diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 72a7a98c2ac4f2..00a18751207e7a 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom.minidom` --- Minimal DOM implementation -===================================================== +:mod:`!xml.dom.minidom` --- Minimal DOM implementation +====================================================== .. module:: xml.dom.minidom :synopsis: Minimal Document Object Model (DOM) implementation. diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index 843c2fd7fdb937..fd96765cbe3c96 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom.pulldom` --- Support for building partial DOM trees -================================================================= +:mod:`!xml.dom.pulldom` --- Support for building partial DOM trees +================================================================== .. module:: xml.dom.pulldom :synopsis: Support for building partial DOM trees from SAX events. diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index d0e1b248d595d1..f33b19bc2724d0 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom` --- The Document Object Model API -================================================ +:mod:`!xml.dom` --- The Document Object Model API +================================================= .. module:: xml.dom :synopsis: Document Object Model API for Python. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 30a7b653f940e9..a6a9eb87f56d88 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1,5 +1,5 @@ -:mod:`xml.etree.ElementTree` --- The ElementTree XML API -======================================================== +:mod:`!xml.etree.ElementTree` --- The ElementTree XML API +========================================================= .. module:: xml.etree.ElementTree :synopsis: Implementation of the ElementTree API. diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index e2f28e3244cb09..c2c9d6424b5072 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.handler` --- Base classes for SAX handlers -======================================================== +:mod:`!xml.sax.handler` --- Base classes for SAX handlers +========================================================= .. module:: xml.sax.handler :synopsis: Base classes for SAX event handlers. diff --git a/Doc/library/xml.sax.reader.rst b/Doc/library/xml.sax.reader.rst index 113e9e93fb04ff..b0bc84062e0719 100644 --- a/Doc/library/xml.sax.reader.rst +++ b/Doc/library/xml.sax.reader.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.xmlreader` --- Interface for XML parsers -====================================================== +:mod:`!xml.sax.xmlreader` --- Interface for XML parsers +======================================================= .. module:: xml.sax.xmlreader :synopsis: Interface which SAX-compliant XML parsers must implement. diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index 6d351dfb4d7072..c60e9e505f7544 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax` --- Support for SAX2 parsers -=========================================== +:mod:`!xml.sax` --- Support for SAX2 parsers +============================================ .. module:: xml.sax :synopsis: Package containing SAX2 base classes and convenience functions. diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index 3a524c9c0d5a9f..5ee11d58c3dd26 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.saxutils` --- SAX Utilities -========================================= +:mod:`!xml.sax.saxutils` --- SAX Utilities +========================================== .. module:: xml.sax.saxutils :synopsis: Convenience functions and classes for use with SAX. diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index f7f23007fb0522..614fb19d1f56b6 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -1,5 +1,5 @@ -:mod:`xmlrpc.client` --- XML-RPC client access -============================================== +:mod:`!xmlrpc.client` --- XML-RPC client access +=============================================== .. module:: xmlrpc.client :synopsis: XML-RPC client access. diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index ca1ea455f0acfc..06169c7eca8b0c 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -1,5 +1,5 @@ -:mod:`xmlrpc.server` --- Basic XML-RPC servers -============================================== +:mod:`!xmlrpc.server` --- Basic XML-RPC servers +=============================================== .. module:: xmlrpc.server :synopsis: Basic XML-RPC server implementations. diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index c8a059bdb1cb93..cf561b454e934f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -1,5 +1,5 @@ -:mod:`zipapp` --- Manage executable Python zip archives -======================================================= +:mod:`!zipapp` --- Manage executable Python zip archives +======================================================== .. module:: zipapp :synopsis: Manage executable Python zip archives diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 6192689c3a1194..aad2028523dc34 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -1,5 +1,5 @@ -:mod:`zipfile` --- Work with ZIP archives -========================================= +:mod:`!zipfile` --- Work with ZIP archives +========================================== .. module:: zipfile :synopsis: Read and write ZIP-format archive files. diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 7a8c837307e60a..9353a45bdcecba 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -1,5 +1,5 @@ -:mod:`zipimport` --- Import modules from Zip archives -===================================================== +:mod:`!zipimport` --- Import modules from Zip archives +====================================================== .. module:: zipimport :synopsis: Support for importing Python modules from ZIP archives. diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index ac179722dee2be..965b82a3daffb9 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -1,5 +1,5 @@ -:mod:`zlib` --- Compression compatible with :program:`gzip` -=========================================================== +:mod:`!zlib` --- Compression compatible with :program:`gzip` +============================================================ .. module:: zlib :synopsis: Low-level interface to compression and decompression routines diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index 54f1988375570c..a57f3b8b3e858c 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -1,5 +1,5 @@ -:mod:`zoneinfo` --- IANA time zone support -========================================== +:mod:`!zoneinfo` --- IANA time zone support +=========================================== .. module:: zoneinfo :synopsis: IANA time zone support From 05c2fe1acda9ea5a57061642c36e8b73bb4fbba4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 8 May 2024 22:35:16 +0300 Subject: [PATCH 011/903] Format None, True, False and NotImplemented as literals (GH-118758) --- Doc/c-api/object.rst | 2 +- Doc/library/_thread.rst | 6 +++--- Doc/library/asyncio-eventloop.rst | 2 +- Doc/library/asyncio-task.rst | 2 +- Doc/library/bdb.rst | 22 +++++++++++----------- Doc/library/code.rst | 2 +- Doc/library/compileall.rst | 6 +++--- Doc/library/csv.rst | 4 ++-- Doc/library/dataclasses.rst | 2 +- Doc/library/functools.rst | 2 +- Doc/library/gc.rst | 2 +- Doc/library/gzip.rst | 2 +- Doc/library/imaplib.rst | 6 +++--- Doc/library/importlib.resources.abc.rst | 4 ++-- Doc/library/logging.handlers.rst | 2 +- Doc/library/math.rst | 2 +- Doc/library/multiprocessing.rst | 8 ++++---- Doc/library/os.rst | 4 ++-- Doc/library/pathlib.rst | 4 ++-- Doc/library/pickle.rst | 16 ++++++++-------- Doc/library/pyclbr.rst | 2 +- Doc/library/statistics.rst | 2 +- Doc/library/sys.rst | 2 +- Doc/library/test.rst | 2 +- Doc/library/tkinter.dnd.rst | 4 ++-- Doc/library/tty.rst | 4 ++-- Doc/library/typing.rst | 2 +- Doc/library/urllib.request.rst | 2 +- Doc/tutorial/datastructures.rst | 2 +- Misc/NEWS.d/3.11.0a1.rst | 4 ++-- Misc/NEWS.d/3.11.0a2.rst | 4 ++-- Misc/NEWS.d/3.11.0b1.rst | 9 +++++---- Misc/NEWS.d/3.12.0a1.rst | 10 +++++----- Misc/NEWS.d/3.12.0a3.rst | 2 +- Misc/NEWS.d/3.12.0a4.rst | 4 ++-- Misc/NEWS.d/3.12.0b1.rst | 2 +- Misc/NEWS.d/3.13.0a1.rst | 3 ++- Misc/NEWS.d/3.13.0a2.rst | 6 +++--- Misc/NEWS.d/3.13.0a3.rst | 2 +- 39 files changed, 85 insertions(+), 83 deletions(-) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index ba454db9117504..8eeac3fc8a1e58 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -65,7 +65,7 @@ Object Protocol Properly handle returning :c:data:`Py_NotImplemented` from within a C function (that is, create a new :term:`strong reference` - to NotImplemented and return it). + to :const:`NotImplemented` and return it). .. c:macro:: Py_PRINT_RAW diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 3842539ce405d9..81f0cac947f602 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -169,14 +169,14 @@ Lock objects have the following methods: time can acquire a lock --- that's their reason for existence). If the *blocking* argument is present, the action depends on its - value: if it is False, the lock is only acquired if it can be acquired - immediately without waiting, while if it is True, the lock is acquired + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired unconditionally as above. If the floating-point *timeout* argument is present and positive, it specifies the maximum wait time in seconds before returning. A negative *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is False. + a *timeout* if *blocking* is false. The return value is ``True`` if the lock is acquired successfully, ``False`` if not. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index d6ed817b13676f..374e789e91e790 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -796,7 +796,7 @@ Creating network servers :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths are supported. - If *cleanup_socket* is True then the Unix socket will automatically + If *cleanup_socket* is true then the Unix socket will automatically be removed from the filesystem when the server is closed, unless the socket has been replaced after the server has been created. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 188649a2968df8..c5deac7e2748ae 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -538,7 +538,7 @@ Running Tasks Concurrently # [2, 6, 24] .. note:: - If *return_exceptions* is False, cancelling gather() after it + If *return_exceptions* is false, cancelling gather() after it has been marked done won't cancel any submitted awaitables. For instance, gather can be marked done after propagating an exception to the caller, therefore, calling ``gather.cancel()`` diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index c873666f325fbc..85df7914a9a014 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -86,7 +86,7 @@ The :mod:`bdb` module also defines two classes: .. attribute:: temporary - True if a :class:`Breakpoint` at (file, line) is temporary. + ``True`` if a :class:`Breakpoint` at (file, line) is temporary. .. attribute:: cond @@ -99,7 +99,7 @@ The :mod:`bdb` module also defines two classes: .. attribute:: enabled - True if :class:`Breakpoint` is enabled. + ``True`` if :class:`Breakpoint` is enabled. .. attribute:: bpbynumber @@ -215,22 +215,22 @@ The :mod:`bdb` module also defines two classes: .. method:: is_skipped_line(module_name) - Return True if *module_name* matches any skip pattern. + Return ``True`` if *module_name* matches any skip pattern. .. method:: stop_here(frame) - Return True if *frame* is below the starting frame in the stack. + Return ``True`` if *frame* is below the starting frame in the stack. .. method:: break_here(frame) - Return True if there is an effective breakpoint for this line. + Return ``True`` if there is an effective breakpoint for this line. Check whether a line or function breakpoint exists and is in effect. Delete temporary breakpoints based on information from :func:`effective`. .. method:: break_anywhere(frame) - Return True if any breakpoint exists for *frame*'s filename. + Return ``True`` if any breakpoint exists for *frame*'s filename. Derived classes should override these methods to gain control over debugger operation. @@ -348,7 +348,7 @@ The :mod:`bdb` module also defines two classes: .. method:: get_break(filename, lineno) - Return True if there is a breakpoint for *lineno* in *filename*. + Return ``True`` if there is a breakpoint for *lineno* in *filename*. .. method:: get_breaks(filename, lineno) @@ -412,7 +412,7 @@ Finally, the module defines the following functions: .. function:: checkfuncname(b, frame) - Return True if we should break here, depending on the way the + Return ``True`` if we should break here, depending on the way the :class:`Breakpoint` *b* was set. If it was set via line number, it checks if @@ -431,14 +431,14 @@ Finally, the module defines the following functions: :attr:`bplist ` for the (:attr:`file `, :attr:`line `) (which must exist) that is :attr:`enabled `, for - which :func:`checkfuncname` is True, and that has neither a False + which :func:`checkfuncname` is true, and that has neither a false :attr:`condition ` nor positive :attr:`ignore ` count. The *flag*, meaning that a - temporary breakpoint should be deleted, is False only when the + temporary breakpoint should be deleted, is ``False`` only when the :attr:`cond ` cannot be evaluated (in which case, :attr:`ignore ` count is ignored). - If no such entry exists, then (None, None) is returned. + If no such entry exists, then ``(None, None)`` is returned. .. function:: set_trace() diff --git a/Doc/library/code.rst b/Doc/library/code.rst index c297160e825c1a..8c3a3e8e95a11f 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -27,7 +27,7 @@ build applications which provide an interactive interpreter prompt. Closely emulate the behavior of the interactive Python interpreter. This class builds on :class:`InteractiveInterpreter` and adds prompting using the familiar - ``sys.ps1`` and ``sys.ps2``, and input buffering. If *local_exit* is True, + ``sys.ps1`` and ``sys.ps2``, and input buffering. If *local_exit* is true, ``exit()`` and ``quit()`` in the console will not raise :exc:`SystemExit`, but instead return to the calling code. diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 8375648e6dd2d3..d9c0cb67a92aa7 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -226,7 +226,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. .. versionchanged:: 3.8 Setting *workers* to 0 now chooses the optimal number of cores. @@ -289,7 +289,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. .. versionchanged:: 3.9 Added *stripdir*, *prependdir*, *limit_sl_dest* and *hardlink_dupes* arguments. @@ -318,7 +318,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 3ed199750affab..533cdf13974be6 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -349,8 +349,8 @@ The :mod:`csv` module defines the following constants: ``None``. This is similar to :data:`QUOTE_ALL`, except that if a field value is ``None`` an empty (unquoted) string is written. - Instructs :class:`reader` objects to interpret an empty (unquoted) field as None and - to otherwise behave as :data:`QUOTE_ALL`. + Instructs :class:`reader` objects to interpret an empty (unquoted) field + as ``None`` and to otherwise behave as :data:`QUOTE_ALL`. .. versionadded:: 3.12 diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 7a8b83e1384e5f..98459c1b6e1020 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -278,7 +278,7 @@ Module contents generated equality and comparison methods (:meth:`~object.__eq__`, :meth:`~object.__gt__`, et al.). - - *metadata*: This can be a mapping or None. None is treated as + - *metadata*: This can be a mapping or ``None``. ``None`` is treated as an empty dict. This value is wrapped in :func:`~types.MappingProxyType` to make it read-only, and exposed on the :class:`Field` object. It is not used at all by Data diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index f3f2f1169f7986..1f003b3a94fda2 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -325,7 +325,7 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.2 .. versionchanged:: 3.4 - Returning NotImplemented from the underlying comparison function for + Returning ``NotImplemented`` from the underlying comparison function for unrecognised types is now supported. .. function:: partial(func, /, *args, **keywords) diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 9e3f942904dc54..790dfdfd00b196 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -69,7 +69,7 @@ The :mod:`gc` module provides the following functions: .. function:: get_objects(generation=None) Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not None, return only the objects tracked by + returned. If *generation* is not ``None``, return only the objects tracked by the collector that are in that generation. .. versionchanged:: 3.8 diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index c609d54b73883e..965da5981f6dbc 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -102,7 +102,7 @@ The module defines the following items: The optional *mtime* argument is the timestamp requested by gzip. The time is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If *mtime* is omitted or None, the current time is used. Use *mtime* = 0 + If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 to generate a compressed stream that does not depend on creation time. See below for the :attr:`mtime` attribute that is set when decompressing. diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 2418a3b6b719e7..a2dad58b00b9fa 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -39,7 +39,7 @@ base class: initialized. If *host* is not specified, ``''`` (the local host) is used. If *port* is omitted, the standard IMAP4 port (143) is used. The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. - If timeout is not given or is None, the global default socket timeout is used. + If timeout is not given or is ``None``, the global default socket timeout is used. The :class:`IMAP4` class supports the :keyword:`with` statement. When used like this, the IMAP4 ``LOGOUT`` command is issued automatically when the @@ -97,7 +97,7 @@ There's also a subclass for secure connections: best practices. The optional *timeout* parameter specifies a timeout in seconds for the - connection attempt. If timeout is not given or is None, the global default + connection attempt. If timeout is not given or is ``None``, the global default socket timeout is used. .. versionchanged:: 3.3 @@ -360,7 +360,7 @@ An :class:`IMAP4` instance has the following methods: Opens socket to *port* at *host*. The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. - If timeout is not given or is None, the global default socket timeout + If timeout is not given or is ``None``, the global default socket timeout is used. Also note that if the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to reject creating a non-blocking socket. This method is implicitly called by the :class:`IMAP4` constructor. diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 7261eacea905f0..5ea8044e1ec6ca 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -103,11 +103,11 @@ .. abstractmethod:: is_dir() - Return True if self is a directory. + Return ``True`` if self is a directory. .. abstractmethod:: is_file() - Return True if self is a file. + Return ``True`` if self is a file. .. abstractmethod:: joinpath(*pathsegments) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 68a54c2ead25bc..5a081f9e7add99 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -66,7 +66,7 @@ and :meth:`flush` methods). :param stream: The stream that the handler should use. - :return: the old stream, if the stream was changed, or *None* if it wasn't. + :return: the old stream, if the stream was changed, or ``None`` if it wasn't. .. versionadded:: 3.7 diff --git a/Doc/library/math.rst b/Doc/library/math.rst index dc43874bf4eabd..316144992d6832 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -268,7 +268,7 @@ Number-theoretic and representation functions Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates to zero when ``k > n``. - If *k* is not specified or is None, then *k* defaults to *n* + If *k* is not specified or is ``None``, then *k* defaults to *n* and the function returns ``n!``. Raises :exc:`TypeError` if either of the arguments are not integers. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index ff42dcf5f5b299..49762491bae5f4 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2483,9 +2483,9 @@ multiple connections at the same time. generally be omitted since it can usually be inferred from the format of *address*. (See :ref:`multiprocessing-address-formats`) - If *authkey* is given and not None, it should be a byte string and will be + If *authkey* is given and not ``None``, it should be a byte string and will be used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. + authentication is done if *authkey* is ``None``. :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. See :ref:`multiprocessing-auth-keys`. @@ -2518,9 +2518,9 @@ multiple connections at the same time. to the :meth:`~socket.socket.listen` method of the socket once it has been bound. - If *authkey* is given and not None, it should be a byte string and will be + If *authkey* is given and not ``None``, it should be a byte string and will be used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. + authentication is done if *authkey* is ``None``. :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. See :ref:`multiprocessing-auth-keys`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index eb558357705ad6..b93b06d4e72afc 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -923,7 +923,7 @@ as internal buffering of data. Copy *count* bytes from file descriptor *src*, starting from offset *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. - If *offset_src* is None, then *src* is read from the current position; + If *offset_src* is ``None``, then *src* is read from the current position; respectively for *offset_dst*. In Linux kernel older than 5.3, the files pointed to by *src* and *dst* @@ -1718,7 +1718,7 @@ or `the MSDN `_ on Windo Transfer *count* bytes from file descriptor *src*, starting from offset *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. At least one of the file descriptors must refer to a pipe. If *offset_src* - is None, then *src* is read from the current position; respectively for + is ``None``, then *src* is read from the current position; respectively for *offset_dst*. The offset associated to the file descriptor that refers to a pipe must be ``None``. The files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 5bf021fbc43210..15e9eaa5190256 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -627,8 +627,8 @@ Pure paths provide the following methods and properties: raise ValueError(error_message.format(str(self), str(formatted))) ValueError: '/etc/passwd' is not in the subpath of '/usr' OR one path is relative and the other is absolute. - When *walk_up* is False (the default), the path must start with *other*. - When the argument is True, ``..`` entries may be added to form the + When *walk_up* is false (the default), the path must start with *other*. + When the argument is true, ``..`` entries may be added to form the relative path. In all other cases, such as the paths referencing different drives, :exc:`ValueError` is raised.:: diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index fe589cf5cc8950..57fbe5b6ece6b6 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -314,16 +314,16 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, map the new Python 3 names to the old module names used in Python 2, so that the pickle data stream is readable with Python 2. - If *buffer_callback* is None (the default), buffer views are + If *buffer_callback* is ``None`` (the default), buffer views are serialized into *file* as part of the pickle stream. - If *buffer_callback* is not None, then it can be called any number + If *buffer_callback* is not ``None``, then it can be called any number of times with a buffer view. If the callback returns a false value - (such as None), the given buffer is :ref:`out-of-band `; + (such as ``None``), the given buffer is :ref:`out-of-band `; otherwise the buffer is serialized in-band, i.e. inside the pickle stream. - It is an error if *buffer_callback* is not None and *protocol* is - None or smaller than 5. + It is an error if *buffer_callback* is not ``None`` and *protocol* is + ``None`` or smaller than 5. .. versionchanged:: 3.8 The *buffer_callback* argument was added. @@ -420,12 +420,12 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, instances of :class:`~datetime.datetime`, :class:`~datetime.date` and :class:`~datetime.time` pickled by Python 2. - If *buffers* is None (the default), then all data necessary for + If *buffers* is ``None`` (the default), then all data necessary for deserialization must be contained in the pickle stream. This means - that the *buffer_callback* argument was None when a :class:`Pickler` + that the *buffer_callback* argument was ``None`` when a :class:`Pickler` was instantiated (or when :func:`dump` or :func:`dumps` was called). - If *buffers* is not None, it should be an iterable of buffer-enabled + If *buffers* is not ``None``, it should be an iterable of buffer-enabled objects that is consumed each time the pickle stream references an :ref:`out-of-band ` buffer view. Such buffers have been given in order to the *buffer_callback* of a Pickler object. diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index da359004e6dd1f..5efb11d89dd143 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -142,7 +142,7 @@ Class Objects .. attribute:: parent - For top-level classes, None. For nested classes, the parent. + For top-level classes, ``None``. For nested classes, the parent. .. versionadded:: 3.7 diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 7af8c5ae618dff..8453135d2e164d 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -220,7 +220,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: harmonic_mean(data, weights=None) Return the harmonic mean of *data*, a sequence or iterable of - real-valued numbers. If *weights* is omitted or *None*, then + real-valued numbers. If *weights* is omitted or ``None``, then equal weighting is assumed. The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index e6d39522fc6da3..ed809d04167ffd 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1708,7 +1708,7 @@ always available. contain a tuple of (filename, line number, function name) tuples describing the traceback where the coroutine object was created, with the most recent call first. When disabled, ``cr_origin`` will - be None. + be ``None``. To enable, pass a *depth* value greater than zero; this sets the number of frames whose information will be captured. To disable, diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 0fff1a0ae3b563..2a61f0aaef2b4c 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -324,7 +324,7 @@ The :mod:`test.support` module defines the following constants: .. data:: Py_DEBUG - True if Python was built with the :c:macro:`Py_DEBUG` macro + ``True`` if Python was built with the :c:macro:`Py_DEBUG` macro defined, that is, if Python was :ref:`built in debug mode `. diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst index ccf698af228d02..62298d96c26459 100644 --- a/Doc/library/tkinter.dnd.rst +++ b/Doc/library/tkinter.dnd.rst @@ -25,8 +25,8 @@ Selection of a target object occurs as follows: #. Top-down search of area under mouse for target widget * Target widget should have a callable *dnd_accept* attribute - * If *dnd_accept* is not present or returns None, search moves to parent widget - * If no target widget is found, then the target object is None + * If *dnd_accept* is not present or returns ``None``, search moves to parent widget + * If no target widget is found, then the target object is ``None`` 2. Call to *.dnd_leave(source, event)* #. Call to *.dnd_enter(source, event)* diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst index 0641d8a68d5e9e..37778bf20bdcc7 100644 --- a/Doc/library/tty.rst +++ b/Doc/library/tty.rst @@ -53,7 +53,7 @@ The :mod:`tty` module defines the following functions: is saved before setting *fd* to raw mode; this value is returned. .. versionchanged:: 3.12 - The return value is now the original tty attributes, instead of None. + The return value is now the original tty attributes, instead of ``None``. .. function:: setcbreak(fd, when=termios.TCSAFLUSH) @@ -67,7 +67,7 @@ The :mod:`tty` module defines the following functions: the minimum input to 1 byte with no delay. .. versionchanged:: 3.12 - The return value is now the original tty attributes, instead of None. + The return value is now the original tty attributes, instead of ``None``. .. versionchanged:: 3.12.2 The ``ICRNL`` flag is no longer cleared. This restores the behavior diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index f53080e0610cb1..812589a3920ecd 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2498,7 +2498,7 @@ types. This attribute reflects *only* the value of the ``total`` argument to the current ``TypedDict`` class, not whether the class is semantically - total. For example, a ``TypedDict`` with ``__total__`` set to True may + total. For example, a ``TypedDict`` with ``__total__`` set to ``True`` may have keys marked with :data:`NotRequired`, or it may inherit from another ``TypedDict`` with ``total=False``. Therefore, it is generally better to use :attr:`__required_keys__` and :attr:`__optional_keys__` for introspection. diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index bd39f2cbc6570c..754405e0fbe5b2 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -218,7 +218,7 @@ The following classes are provided: An appropriate ``Content-Type`` header should be included if the *data* argument is present. If this header has not been provided and *data* - is not None, ``Content-Type: application/x-www-form-urlencoded`` will + is not ``None``, ``Content-Type: application/x-www-form-urlencoded`` will be added as a default. The next two arguments are only of interest for correct handling diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index de2827461e2f24..a1492298bdb867 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -126,7 +126,7 @@ Python. Another thing you might notice is that not all data can be sorted or compared. For instance, ``[None, 'hello', 10]`` doesn't sort because -integers can't be compared to strings and *None* can't be compared to +integers can't be compared to strings and ``None`` can't be compared to other types. Also, there are some types that don't have a defined ordering relation. For example, ``3+4j < 5+7j`` isn't a valid comparison. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index e6cf9c001a1a01..96a7cf6984dd9e 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -2722,7 +2722,7 @@ Importing typing.io or typing.re now prints a ``DeprecationWarning``. .. section: Library argparse actions store_const and append_const each receive a default value -of None when the ``const`` kwarg is not provided. Previously, this raised a +of ``None`` when the ``const`` kwarg is not provided. Previously, this raised a :exc:`TypeError`. .. @@ -3995,7 +3995,7 @@ operator expressions. .. section: Documentation Document that :class:`collections.defaultdict` parameter ``default_factory`` -defaults to None and is positional-only. +defaults to ``None`` and is positional-only. .. diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index a6b5fe54b391c5..f3fc62e9097162 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -331,8 +331,8 @@ underlying SQLite API signals memory error. Patch by Erlend E. Aasland. .. nonce: 4MQt4r .. section: Library -pprint.pprint() now handles underscore_numbers correctly. Previously it was -always setting it to False. +:func:`pprint.pprint` now handles *underscore_numbers* correctly. +Previously it was always setting it to ``False``. .. diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index f9296679655573..c35e8e2c1caf07 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -58,10 +58,10 @@ may have prevented Python-to-Python calls respecting PEP 523. .. nonce: -igcjS .. section: Core and Builtins -Add a closure keyword-only parameter to exec(). It can only be specified +Add a closure keyword-only parameter to :func:`exec()`. It can only be specified when exec-ing a code object that uses free variables. When specified, it must be a tuple, with exactly the number of cell variables referenced by the -code object. closure has a default value of None, and it must be None if the +code object. closure has a default value of ``None``, and it must be ``None`` if the code object doesn't refer to any free variables. .. @@ -664,8 +664,9 @@ for :func:`os.fcopyfile` available in macOs. .. nonce: l1p7CJ .. section: Library -For @dataclass, add weakref_slot. Default is False. If True, and if -slots=True, add a slot named "__weakref__", which will allow instances to be +For :func:`@dataclass `, add *weakref_slot*. +The new parameter defaults to ``False``. If true, and if +``slots=True``, add a slot named ``"__weakref__"``, which will allow instances to be weakref'd. Contributed by Eric V. Smith .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index f75a83c1d950d4..f2438d6608b7af 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -3561,8 +3561,8 @@ with :func:`os.pidfd_open` in non-blocking mode. Patch by Kumar Aditya. .. nonce: mkYl5q .. section: Library -Implement Enum __contains__ that returns True or False to replace the -deprecated behaviour that would sometimes raise a TypeError. +Implement ``Enum.__contains__`` that returns ``True`` or ``False`` to replace the +deprecated behaviour that would sometimes raise a :exc:`TypeError`. .. @@ -3729,7 +3729,7 @@ In a very special case, the email package tried to append the nonexistent .. nonce: e6uKxj .. section: Library -Fix :func:`ast.unparse` when ``ImportFrom.level`` is None +Fix :func:`ast.unparse` when ``ImportFrom.level`` is ``None`` .. @@ -3791,7 +3791,7 @@ the :c:type:`time_t` type in C. .. section: Library Fixed crash resulting from calling bisect.insort() or bisect.insort_left() -with the key argument not equal to None. +with the key argument not equal to ``None``. .. @@ -4080,7 +4080,7 @@ replacement strings containing group references by 2--3 times. .. section: Library Fix findtext in the xml module to only give an empty string when the text -attribute is set to None. +attribute is set to ``None``. .. diff --git a/Misc/NEWS.d/3.12.0a3.rst b/Misc/NEWS.d/3.12.0a3.rst index ce128fd5f80c77..9b789c68607c73 100644 --- a/Misc/NEWS.d/3.12.0a3.rst +++ b/Misc/NEWS.d/3.12.0a3.rst @@ -496,7 +496,7 @@ Created packages from zipfile and test_zipfile modules, separating Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the :mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would -return True from this method; now they correctly return False. +return ``True`` from this method; now they correctly return ``False``. .. diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index 82faa5ad0b2031..1599084ae0d323 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -65,8 +65,8 @@ redundant. .. nonce: M2n6Kg .. section: Core and Builtins -Fix :func:`int.__sizeof__` calculation to include the 1 element ob_digit -array for 0 and False. +Fix :func:`int.__sizeof__` calculation to include the 1-element ``ob_digit`` +array for ``0`` and ``False``. .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 764b80b7b7d436..a264f7fad9e65a 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -395,7 +395,7 @@ Fix bug in line numbers of instructions emitted for :keyword:`except* .. section: Core and Builtins Clarify :exc:`SyntaxWarning` with literal ``is`` comparison by specifying -which literal is problematic, since comparisons using ``is`` with e.g. None +which literal is problematic, since comparisons using ``is`` with e.g. ``None`` and bool literals are idiomatic. .. diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 4937f9da5ae629..0092db29460c37 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -3436,7 +3436,8 @@ added support for this decorator. Patch by Alex Waygood. .. nonce: C1ahtk .. section: Library -Make pydoc.doc catch bad module ImportError when output stream is not None. +Make :func:`pydoc.doc` catch bad module :exc:`ImportError` +when output stream is not ``None``. .. diff --git a/Misc/NEWS.d/3.13.0a2.rst b/Misc/NEWS.d/3.13.0a2.rst index e5841e14c02efb..2480ee8202d0f4 100644 --- a/Misc/NEWS.d/3.13.0a2.rst +++ b/Misc/NEWS.d/3.13.0a2.rst @@ -228,7 +228,7 @@ cross-interpreter API. Patch by Anthony Shaw. .. nonce: ageUWQ .. section: Core and Builtins -Add support for sharing of True and False between interpreters using the +Add support for sharing of ``True`` and ``False`` between interpreters using the cross-interpreter API. Patch by Anthony Shaw. .. @@ -1354,8 +1354,8 @@ crash encountered after the first :meth:`tkinter.Tk` instance is destroyed. .. section: IDLE Add docstrings to the IDLE debugger module. Fix two bugs: initialize -Idb.botframe (should be in Bdb); in Idb.in_rpc_code, check whether -prev_frame is None before trying to use it. Greatly expand test_debugger. +``Idb.botframe`` (should be in Bdb); in ``Idb.in_rpc_code``, check whether +``prev_frame`` is ``None`` before trying to use it. Greatly expand test_debugger. .. diff --git a/Misc/NEWS.d/3.13.0a3.rst b/Misc/NEWS.d/3.13.0a3.rst index 218ba609bd80c0..c2fe9dc1ad5aa0 100644 --- a/Misc/NEWS.d/3.13.0a3.rst +++ b/Misc/NEWS.d/3.13.0a3.rst @@ -269,7 +269,7 @@ Correctly compute end column offsets for multiline tokens in the .. nonce: 4ADN7i .. section: Core and Builtins -Fix None.__ne__(None) returning NotImplemented instead of False +Fix ``None.__ne__(None)`` returning ``NotImplemented`` instead of ``False``. .. From 8d84120b4175daaf4ae6038621f3005cdb65acda Mon Sep 17 00:00:00 2001 From: Yutian Li Date: Wed, 8 May 2024 15:58:48 -0400 Subject: [PATCH 012/903] Fixing a typo in test_cmd_line.py (#118728) --- Lib/test/test_cmd_line.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 9624d35d0c3948..058470082fbbf0 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -981,7 +981,7 @@ def test_cpu_count_default(self): self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHON_CPU_COUNT='1234') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - es = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') + res = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) def res2int(self, res): From cb6f75a32ca2649c6cc1cabb0301eb783efbd55b Mon Sep 17 00:00:00 2001 From: mpage Date: Wed, 8 May 2024 13:03:39 -0700 Subject: [PATCH 013/903] gh-117657: Fix data races when writing / reading `ob_gc_bits` (#118292) Use relaxed atomics when reading / writing to the field. There are still a few places in the GC where we do not use atomics. Those should be safe as the world is stopped. --- Include/internal/pycore_gc.h | 50 ++++++++++++++++++---- Include/internal/pycore_object.h | 6 +-- Objects/object.c | 2 +- Tools/tsan/suppressions_free_threading.txt | 3 -- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 281094df786735..60582521db5bd7 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -37,7 +37,15 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { } -/* Bit flags for ob_gc_bits (in Py_GIL_DISABLED builds) */ +/* Bit flags for ob_gc_bits (in Py_GIL_DISABLED builds) + * + * Setting the bits requires a relaxed store. The per-object lock must also be + * held, except when the object is only visible to a single thread (e.g. during + * object initialization or destruction). + * + * Reading the bits requires using a relaxed load, but does not require holding + * the per-object lock. + */ #ifdef Py_GIL_DISABLED # define _PyGC_BITS_TRACKED (1) // Tracked by the GC # define _PyGC_BITS_FINALIZED (2) // tp_finalize was called @@ -48,10 +56,34 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { # define _PyGC_BITS_DEFERRED (64) // Use deferred reference counting #endif +#ifdef Py_GIL_DISABLED + +static inline void +_PyObject_SET_GC_BITS(PyObject *op, uint8_t new_bits) +{ + uint8_t bits = _Py_atomic_load_uint8_relaxed(&op->ob_gc_bits); + _Py_atomic_store_uint8_relaxed(&op->ob_gc_bits, bits | new_bits); +} + +static inline int +_PyObject_HAS_GC_BITS(PyObject *op, uint8_t bits) +{ + return (_Py_atomic_load_uint8_relaxed(&op->ob_gc_bits) & bits) != 0; +} + +static inline void +_PyObject_CLEAR_GC_BITS(PyObject *op, uint8_t bits_to_clear) +{ + uint8_t bits = _Py_atomic_load_uint8_relaxed(&op->ob_gc_bits); + _Py_atomic_store_uint8_relaxed(&op->ob_gc_bits, bits & ~bits_to_clear); +} + +#endif + /* True if the object is currently tracked by the GC. */ static inline int _PyObject_GC_IS_TRACKED(PyObject *op) { #ifdef Py_GIL_DISABLED - return (op->ob_gc_bits & _PyGC_BITS_TRACKED) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_TRACKED); #else PyGC_Head *gc = _Py_AS_GC(op); return (gc->_gc_next != 0); @@ -80,12 +112,12 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { * for calling _PyMem_FreeDelayed on the referenced * memory. */ static inline int _PyObject_GC_IS_SHARED(PyObject *op) { - return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_SHARED); } #define _PyObject_GC_IS_SHARED(op) _PyObject_GC_IS_SHARED(_Py_CAST(PyObject*, op)) static inline void _PyObject_GC_SET_SHARED(PyObject *op) { - op->ob_gc_bits |= _PyGC_BITS_SHARED; + _PyObject_SET_GC_BITS(op, _PyGC_BITS_SHARED); } #define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op)) @@ -95,13 +127,13 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { * Objects with this bit that are GC objects will automatically * delay-freed by PyObject_GC_Del. */ static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) { - return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_SHARED_INLINE); } #define _PyObject_GC_IS_SHARED_INLINE(op) \ _PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op)) static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) { - op->ob_gc_bits |= _PyGC_BITS_SHARED_INLINE; + _PyObject_SET_GC_BITS(op, _PyGC_BITS_SHARED_INLINE); } #define _PyObject_GC_SET_SHARED_INLINE(op) \ _PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op)) @@ -178,7 +210,7 @@ static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) { static inline int _PyGC_FINALIZED(PyObject *op) { #ifdef Py_GIL_DISABLED - return (op->ob_gc_bits & _PyGC_BITS_FINALIZED) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_FINALIZED); #else PyGC_Head *gc = _Py_AS_GC(op); return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0); @@ -186,7 +218,7 @@ static inline int _PyGC_FINALIZED(PyObject *op) { } static inline void _PyGC_SET_FINALIZED(PyObject *op) { #ifdef Py_GIL_DISABLED - op->ob_gc_bits |= _PyGC_BITS_FINALIZED; + _PyObject_SET_GC_BITS(op, _PyGC_BITS_FINALIZED); #else PyGC_Head *gc = _Py_AS_GC(op); gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED; @@ -194,7 +226,7 @@ static inline void _PyGC_SET_FINALIZED(PyObject *op) { } static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { #ifdef Py_GIL_DISABLED - op->ob_gc_bits &= ~_PyGC_BITS_FINALIZED; + _PyObject_CLEAR_GC_BITS(op, _PyGC_BITS_FINALIZED); #else PyGC_Head *gc = _Py_AS_GC(op); gc->_gc_prev &= ~_PyGC_PREV_MASK_FINALIZED; diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index f15c332a7b0811..7602248f956405 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -168,7 +168,7 @@ static inline int _PyObject_HasDeferredRefcount(PyObject *op) { #ifdef Py_GIL_DISABLED - return (op->ob_gc_bits & _PyGC_BITS_DEFERRED) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_DEFERRED); #else return 0; #endif @@ -320,7 +320,7 @@ static inline void _PyObject_GC_TRACK( "object already tracked by the garbage collector", filename, lineno, __func__); #ifdef Py_GIL_DISABLED - op->ob_gc_bits |= _PyGC_BITS_TRACKED; + _PyObject_SET_GC_BITS(op, _PyGC_BITS_TRACKED); #else PyGC_Head *gc = _Py_AS_GC(op); _PyObject_ASSERT_FROM(op, @@ -361,7 +361,7 @@ static inline void _PyObject_GC_UNTRACK( filename, lineno, __func__); #ifdef Py_GIL_DISABLED - op->ob_gc_bits &= ~_PyGC_BITS_TRACKED; + _PyObject_CLEAR_GC_BITS(op, _PyGC_BITS_TRACKED); #else PyGC_Head *gc = _Py_AS_GC(op); PyGC_Head *prev = _PyGCHead_PREV(gc); diff --git a/Objects/object.c b/Objects/object.c index 29183dcc5b4a01..d4fe14c5b3d1aa 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2427,6 +2427,7 @@ _PyObject_SetDeferredRefcount(PyObject *op) assert(PyType_IS_GC(Py_TYPE(op))); assert(_Py_IsOwnedByCurrentThread(op)); assert(op->ob_ref_shared == 0); + _PyObject_SET_GC_BITS(op, _PyGC_BITS_DEFERRED); PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->gc.immortalize.enabled) { // gh-117696: immortalize objects instead of using deferred reference @@ -2434,7 +2435,6 @@ _PyObject_SetDeferredRefcount(PyObject *op) _Py_SetImmortal(op); return; } - op->ob_gc_bits |= _PyGC_BITS_DEFERRED; op->ob_ref_local += 1; op->ob_ref_shared = _Py_REF_QUEUED; #endif diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 74dbf4bb1cb688..a669bc4d1d5c30 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -19,9 +19,6 @@ race:_PyImport_AcquireLock race:_PyImport_ReleaseLock race:_PyInterpreterState_SetNotRunningMain race:_PyInterpreterState_IsRunningMain -race:_PyObject_GC_IS_SHARED -race:_PyObject_GC_SET_SHARED -race:_PyObject_GC_TRACK # https://gist.github.com/mpage/0a24eb2dd458441ededb498e9b0e5de8 race:_PyParkingLot_Park race:_PyType_HasFeature From 2f4db5a04d6fa7ba5c1c6b82b482dd7ca48f3382 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 9 May 2024 02:37:55 +0300 Subject: [PATCH 014/903] gh-118803: Remove `ByteString` from `typing` and `collections.abc` (#118804) --- Doc/library/collections.abc.rst | 10 ------ Doc/library/stdtypes.rst | 1 - Doc/library/typing.rst | 12 ------- Doc/whatsnew/3.12.rst | 4 +-- Doc/whatsnew/3.13.rst | 4 +-- Doc/whatsnew/3.14.rst | 3 ++ Lib/_collections_abc.py | 36 ++----------------- Lib/test/libregrtest/refleak.py | 1 - Lib/test/test_collections.py | 24 +------------ Lib/test/test_typing.py | 11 ------ Lib/typing.py | 19 ---------- ...-05-09-00-52-30.gh-issue-118803.Wv3AvU.rst | 3 ++ 12 files changed, 14 insertions(+), 114 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-09-00-52-30.gh-issue-118803.Wv3AvU.rst diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index ea27436f67f0bf..a63a55caa0f66d 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -141,9 +141,6 @@ ABC Inherits from Abstract Methods Mi ``__len__``, ``insert`` -:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods - ``__len__`` - :class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` @@ -257,7 +254,6 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Sequence MutableSequence - ByteString ABCs for read-only and mutable :term:`sequences `. @@ -274,12 +270,6 @@ Collections Abstract Base Classes -- Detailed Descriptions The index() method added support for *stop* and *start* arguments. - .. deprecated-removed:: 3.12 3.14 - The :class:`ByteString` ABC has been deprecated. - For use in typing, prefer a union, like ``bytes | bytearray``, or - :class:`collections.abc.Buffer`. - For use as an ABC, prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - .. class:: Set MutableSet diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8f5f9ee519a573..f97597dc565449 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5061,7 +5061,6 @@ list is non-exhaustive. * :class:`collections.abc.MutableMapping` * :class:`collections.abc.Sequence` * :class:`collections.abc.MutableSequence` -* :class:`collections.abc.ByteString` * :class:`collections.abc.MappingView` * :class:`collections.abc.KeysView` * :class:`collections.abc.ItemsView` diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 812589a3920ecd..5b74dd9c9f4914 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3502,14 +3502,6 @@ Aliases to container ABCs in :mod:`collections.abc` :class:`collections.abc.Set` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. -.. class:: ByteString(Sequence[int]) - - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview` of byte sequences. - - .. deprecated-removed:: 3.9 3.14 - Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. - .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) Deprecated alias to :class:`collections.abc.Collection`. @@ -3875,10 +3867,6 @@ convenience. This is subject to change, and not all deprecations are listed. - 3.9 - Undecided (see :ref:`deprecated-aliases` for more information) - :pep:`585` - * - :class:`typing.ByteString` - - 3.9 - - 3.14 - - :gh:`91896` * - :data:`typing.Text` - 3.11 - Undecided diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8757311a484257..685d0120f2f636 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1183,7 +1183,7 @@ Deprecated replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) -* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. +* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. (Contributed by Shantanu Jain in :gh:`91896`.) @@ -1293,7 +1293,7 @@ Deprecated :class:`collections.abc.Hashable` and :class:`collections.abc.Sized` respectively, are deprecated. (:gh:`94309`.) - * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a + * :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. (Contributed by Alex Waygood in :gh:`91896`.) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 8e90faee667ded..44555718184e19 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1573,7 +1573,7 @@ Pending Removal in Python 3.14 Use :class:`ast.Constant` instead. (Contributed by Serhiy Storchaka in :gh:`90953`.) -* :mod:`collections.abc`: Deprecated :class:`~collections.abc.ByteString`. +* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. @@ -1647,7 +1647,7 @@ Pending Removal in Python 3.14 May be removed in 3.14. (Contributed by Nikita Sobolev in :gh:`101866`.) -* :mod:`typing`: :class:`~typing.ByteString`, deprecated since Python 3.9, +* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. * :mod:`urllib`: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 6a9d0b0d912fb2..5aa6e4f7a9ed3e 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -105,6 +105,9 @@ Removed It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed by Jelle Zijlstra in :gh:`118767`.) +* :class:`!typing.ByteString` and :class:`!collections.abc.ByteString` + are removed. They had previously raised a :exc:`DeprecationWarning` + since Python 3.12. Porting to Python 3.14 ====================== diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 601107d2d86771..1135e17e379059 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", "Buffer", + "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -1068,40 +1068,10 @@ def count(self, value): Sequence.register(tuple) Sequence.register(str) +Sequence.register(bytes) Sequence.register(range) Sequence.register(memoryview) -class _DeprecateByteStringMeta(ABCMeta): - def __new__(cls, name, bases, namespace, **kwargs): - if name != "ByteString": - import warnings - - warnings._deprecated( - "collections.abc.ByteString", - remove=(3, 14), - ) - return super().__new__(cls, name, bases, namespace, **kwargs) - - def __instancecheck__(cls, instance): - import warnings - - warnings._deprecated( - "collections.abc.ByteString", - remove=(3, 14), - ) - return super().__instancecheck__(instance) - -class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): - """This unifies bytes and bytearray. - - XXX Should add all their methods. - """ - - __slots__ = () - -ByteString.register(bytes) -ByteString.register(bytearray) - class MutableSequence(Sequence): """All the operations on a read-write sequence. @@ -1170,4 +1140,4 @@ def __iadd__(self, values): MutableSequence.register(list) -MutableSequence.register(bytearray) # Multiply inheriting, see ByteString +MutableSequence.register(bytearray) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index f582c0d3e7ff13..bb57ffd75636cb 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -211,7 +211,6 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): zipimport._zip_directory_cache.update(zdc) # Clear ABC registries, restoring previously saved ABC registries. - # ignore deprecation warning for collections.abc.ByteString abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 955323cae88f92..a24d3e3ea142b7 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -26,7 +26,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString, Buffer +from collections.abc import Buffer class TestUserObjects(unittest.TestCase): @@ -1935,28 +1935,6 @@ def assert_index_same(seq1, seq2, index_args): assert_index_same( nativeseq, seqseq, (letter, start, stop)) - def test_ByteString(self): - for sample in [bytes, bytearray]: - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(sample(), ByteString) - self.assertTrue(issubclass(sample, ByteString)) - for sample in [str, list, tuple]: - with self.assertWarns(DeprecationWarning): - self.assertNotIsInstance(sample(), ByteString) - self.assertFalse(issubclass(sample, ByteString)) - with self.assertWarns(DeprecationWarning): - self.assertNotIsInstance(memoryview(b""), ByteString) - self.assertFalse(issubclass(memoryview, ByteString)) - with self.assertWarns(DeprecationWarning): - self.validate_abstract_methods(ByteString, '__getitem__', '__len__') - - with self.assertWarns(DeprecationWarning): - class X(ByteString): pass - - with self.assertWarns(DeprecationWarning): - # No metaclass conflict - class Z(ByteString, Awaitable): pass - def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index fff81f7997764d..8dde8aedd2a2b1 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7128,16 +7128,6 @@ def test_mutablesequence(self): self.assertIsInstance([], typing.MutableSequence) self.assertNotIsInstance((), typing.MutableSequence) - def test_bytestring(self): - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(b'', typing.ByteString) - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(bytearray(b''), typing.ByteString) - with self.assertWarns(DeprecationWarning): - class Foo(typing.ByteString): ... - with self.assertWarns(DeprecationWarning): - class Bar(typing.ByteString, typing.Awaitable): ... - def test_list(self): self.assertIsSubclass(list, typing.List) @@ -9951,7 +9941,6 @@ def test_special_attrs(self): typing.AsyncIterable: 'AsyncIterable', typing.AsyncIterator: 'AsyncIterator', typing.Awaitable: 'Awaitable', - typing.ByteString: 'ByteString', typing.Callable: 'Callable', typing.ChainMap: 'ChainMap', typing.Collection: 'Collection', diff --git a/Lib/typing.py b/Lib/typing.py index c8649e312ddb00..e75a627d226e50 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -64,7 +64,6 @@ # ABCs (from collections.abc). 'AbstractSet', # collections.abc.Set. - 'ByteString', 'Container', 'ContextManager', 'Hashable', @@ -1670,21 +1669,6 @@ def __ror__(self, left): return Union[left, self] -class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): - def __init__( - self, origin, nparams, *, removal_version, inst=True, name=None - ): - super().__init__(origin, nparams, inst=inst, name=name) - self._removal_version = removal_version - - def __instancecheck__(self, inst): - import warnings - warnings._deprecated( - f"{self.__module__}.{self._name}", remove=self._removal_version - ) - return super().__instancecheck__(inst) - - class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' @@ -2828,9 +2812,6 @@ class Other(Leaf): # Error reported by type checker MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) -ByteString = _DeprecatedGenericAlias( - collections.abc.ByteString, 0, removal_version=(3, 14) # Not generic. -) # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ diff --git a/Misc/NEWS.d/next/Library/2024-05-09-00-52-30.gh-issue-118803.Wv3AvU.rst b/Misc/NEWS.d/next/Library/2024-05-09-00-52-30.gh-issue-118803.Wv3AvU.rst new file mode 100644 index 00000000000000..2d86dff57faf72 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-00-52-30.gh-issue-118803.Wv3AvU.rst @@ -0,0 +1,3 @@ +:class:`!typing.ByteString` and :class:`!collections.abc.ByteString` are +removed. They had previously raised a :exc:`DeprecationWarning` since Python +3.12. From 027e6d88fb898b7477b822b84f791ca60e64300b Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 9 May 2024 11:16:37 +0300 Subject: [PATCH 015/903] [tests]: Mark ``test_statistics.test_kde_random`` with a ``requires_resource('cpu')`` decorator (#118801) Mark test_kde_random with a requires_resource('cpu') decorator --- Lib/test/test_statistics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 40680759d456ac..6f68edd447c953 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2446,6 +2446,7 @@ def test_kde_kernel_invcdfs(self): for x in xarr: self.assertAlmostEqual(invcdf(cdf(x)), x, places=5) + @support.requires_resource('cpu') def test_kde_random(self): kde_random = statistics.kde_random StatisticsError = statistics.StatisticsError From e8cbcf49555c694975a6af56b5cb0af7817e889e Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 9 May 2024 03:31:47 -0500 Subject: [PATCH 016/903] GH-101588: Remove deprecated pickle/copy/deepcopy from itertools (gh-118816) --- Doc/whatsnew/3.14.rst | 5 + Lib/test/test_itertools.py | 437 +-------- ...-05-09-02-43-37.gh-issue-101588.30bNAr.rst | 2 + Modules/itertoolsmodule.c | 858 ------------------ 4 files changed, 8 insertions(+), 1294 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-09-02-43-37.gh-issue-101588.30bNAr.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 5aa6e4f7a9ed3e..496a5d8ae7bc5e 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -109,6 +109,11 @@ Removed are removed. They had previously raised a :exc:`DeprecationWarning` since Python 3.12. +* :mod:`itertools` support for copy, deepcopy, and pickle operations. + These had previously raised a :exc:`DeprecationWarning` since Python 3.12. + (Contributed by Raymond Hettinger in :gh:`101588`.) + + Porting to Python 3.14 ====================== diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index e243da309f03d8..4d2c01886724a8 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -144,7 +144,6 @@ def expand(it, i=0): c = expand(compare[took:]) self.assertEqual(a, c); - @pickle_deprecated def test_accumulate(self): self.assertEqual(list(accumulate(range(10))), # one positional arg [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]) @@ -171,9 +170,6 @@ def test_accumulate(self): [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0]) with self.assertRaises(TypeError): list(accumulate(s, chr)) # unary-operation - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, accumulate(range(10))) # test pickling - self.pickletest(proto, accumulate(range(10), initial=7)) self.assertEqual(list(accumulate([10, 5, 1], initial=None)), [10, 15, 16]) self.assertEqual(list(accumulate([10, 5, 1], initial=100)), [100, 110, 115, 116]) self.assertEqual(list(accumulate([], initial=100)), [100]) @@ -245,58 +241,12 @@ def test_chain_from_iterable(self): self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) self.assertEqual(list(islice(chain.from_iterable(repeat(range(5))), 2)), [0, 1]) - @pickle_deprecated - def test_chain_reducible(self): - for oper in [copy.deepcopy] + picklecopiers: - it = chain('abc', 'def') - self.assertEqual(list(oper(it)), list('abcdef')) - self.assertEqual(next(it), 'a') - self.assertEqual(list(oper(it)), list('bcdef')) - - self.assertEqual(list(oper(chain(''))), []) - self.assertEqual(take(4, oper(chain('abc', 'def'))), list('abcd')) - self.assertRaises(TypeError, list, oper(chain(2, 3))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef')) - - @pickle_deprecated - def test_chain_setstate(self): - self.assertRaises(TypeError, chain().__setstate__, ()) - self.assertRaises(TypeError, chain().__setstate__, []) - self.assertRaises(TypeError, chain().__setstate__, 0) - self.assertRaises(TypeError, chain().__setstate__, ([],)) - self.assertRaises(TypeError, chain().__setstate__, (iter([]), [])) - it = chain() - it.__setstate__((iter(['abc', 'def']),)) - self.assertEqual(list(it), ['a', 'b', 'c', 'd', 'e', 'f']) - it = chain() - it.__setstate__((iter(['abc', 'def']), iter(['ghi']))) - self.assertEqual(list(it), ['ghi', 'a', 'b', 'c', 'd', 'e', 'f']) - - @pickle_deprecated def test_combinations(self): self.assertRaises(TypeError, combinations, 'abc') # missing r argument self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments self.assertRaises(TypeError, combinations, None) # pool is not iterable self.assertRaises(ValueError, combinations, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(combinations('abc', 32))), []) # r > n - - self.assertEqual(list(op(combinations('ABCD', 2))), - [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - testIntermediate = combinations('ABCD', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - - self.assertEqual(list(op(combinations(range(4), 3))), - [(0,1,2), (0,1,3), (0,2,3), (1,2,3)]) - testIntermediate = combinations(range(4), 3) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [(0,1,3), (0,2,3), (1,2,3)]) - def combinations1(iterable, r): 'Pure python version shown in the docs' pool = tuple(iterable) @@ -350,9 +300,6 @@ def combinations3(iterable, r): self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, combinations(values, r)) # test pickling - @support.bigaddrspacetest def test_combinations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -364,7 +311,6 @@ def test_combinations_tuple_reuse(self): self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) - @pickle_deprecated def test_combinations_with_replacement(self): cwr = combinations_with_replacement self.assertRaises(TypeError, cwr, 'abc') # missing r argument @@ -372,15 +318,6 @@ def test_combinations_with_replacement(self): self.assertRaises(TypeError, cwr, None) # pool is not iterable self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(cwr('ABC', 2))), - [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - testIntermediate = cwr('ABC', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - - def cwr1(iterable, r): 'Pure python version shown in the docs' # number items returned: (n+r-1)! / r! / (n-1)! when n>0 @@ -438,22 +375,18 @@ def numcombs(n, r): self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cwr(values,r)) # test pickling - @support.bigaddrspacetest def test_combinations_with_replacement_overflow(self): with self.assertRaises((OverflowError, MemoryError)): combinations_with_replacement("AA", 2**30) - # Test implementation detail: tuple re-use + # Test implementation detail: tuple re-use @support.impl_detail("tuple reuse is specific to CPython") def test_combinations_with_replacement_tuple_reuse(self): cwr = combinations_with_replacement self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) - @pickle_deprecated def test_permutations(self): self.assertRaises(TypeError, permutations) # too few arguments self.assertRaises(TypeError, permutations, 'abc', 2, 1) # too many arguments @@ -514,9 +447,6 @@ def permutations2(iterable, r=None): self.assertEqual(result, list(permutations(values, None))) # test r as None self.assertEqual(result, list(permutations(values))) # test default r - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, permutations(values, r)) # test pickling - @support.bigaddrspacetest def test_permutations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -560,7 +490,6 @@ def test_combinatorics(self): self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm - @pickle_deprecated def test_compress(self): self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) self.assertEqual(list(compress('ABCDEF', [1,0,1,0,1,1])), list('ACEF')) @@ -577,24 +506,6 @@ def test_compress(self): self.assertRaises(TypeError, compress, range(6)) # too few args self.assertRaises(TypeError, compress, range(6), None) # too many args - # check copy, deepcopy, pickle - for op in [lambda a:copy.copy(a), lambda a:copy.deepcopy(a)] + picklecopiers: - for data, selectors, result1, result2 in [ - ('ABCDEF', [1,0,1,0,1,1], 'ACEF', 'CEF'), - ('ABCDEF', [0,0,0,0,0,0], '', ''), - ('ABCDEF', [1,1,1,1,1,1], 'ABCDEF', 'BCDEF'), - ('ABCDEF', [1,0,1], 'AC', 'C'), - ('ABC', [0,1,1,1,1,1], 'BC', 'C'), - ]: - - self.assertEqual(list(op(compress(data=data, selectors=selectors))), list(result1)) - self.assertEqual(list(op(compress(data, selectors))), list(result1)) - testIntermediate = compress(data, selectors) - if result1: - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), list(result2)) - - @pickle_deprecated def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)]) @@ -632,18 +543,9 @@ def test_count(self): r2 = 'count(%r)'.__mod__(i) self.assertEqual(r1, r2) - # check copy, deepcopy, pickle - for value in -3, 3, maxsize-5, maxsize+5: - c = count(value) - self.assertEqual(next(copy.copy(c)), value) - self.assertEqual(next(copy.deepcopy(c)), value) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(value)) - #check proper internal error handling for large "step' sizes count(1, maxsize+5); sys.exc_info() - @pickle_deprecated def test_count_with_stride(self): self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) self.assertEqual(lzip('abc',count(start=2,step=3)), @@ -687,17 +589,6 @@ def test_count_with_stride(self): c = count(10, 1.0) self.assertEqual(type(next(c)), int) self.assertEqual(type(next(c)), float) - for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): - for j in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 1, 10, sys.maxsize-5, sys.maxsize+5): - # Test repr - r1 = repr(count(i, j)) - if j == 1: - r2 = ('count(%r)' % i) - else: - r2 = ('count(%r, %r)' % (i, j)) - self.assertEqual(r1, r2) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(i, j)) def test_cycle(self): self.assertEqual(take(10, cycle('abc')), list('abcabcabca')) @@ -706,113 +597,6 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) - @pickle_deprecated - def test_cycle_copy_pickle(self): - # check copy, deepcopy, pickle - c = cycle('abc') - self.assertEqual(next(c), 'a') - #simple copy currently not supported, because __reduce__ returns - #an internal iterator - #self.assertEqual(take(10, copy.copy(c)), list('bcabcabcab')) - self.assertEqual(take(10, copy.deepcopy(c)), list('bcabcabcab')) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('bcabcabcab')) - next(c) - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('cabcabcabc')) - next(c) - next(c) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cycle('abc')) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - # test with partial consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(2)] # consume 2 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - # test with completely consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(7)] # consume 7 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - @pickle_deprecated - def test_cycle_unpickle_compat(self): - testcases = [ - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - ] - assert len(testcases) == 20 - for t in testcases: - it = pickle.loads(t) - self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) - - @pickle_deprecated - def test_cycle_setstate(self): - # Verify both modes for restoring state - - # Mode 0 is efficient. It uses an incompletely consumed input - # iterator to build a cycle object and then passes in state with - # a list of previously consumed values. There is no data - # overlap between the two. - c = cycle('defg') - c.__setstate__((list('abc'), 0)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # Mode 1 is inefficient. It starts with a cycle object built - # from an iterator over the remaining elements in a partial - # cycle and then passes in state with all of the previously - # seen values (this overlaps values included in the iterator). - c = cycle('defg') - c.__setstate__((list('abcdefg'), 1)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # The first argument to setstate needs to be a tuple - with self.assertRaises(TypeError): - cycle('defg').__setstate__([list('abcdefg'), 0]) - - # The first argument in the setstate tuple must be a list - with self.assertRaises(TypeError): - c = cycle('defg') - c.__setstate__((tuple('defg'), 0)) - take(20, c) - - # The second argument in the setstate tuple must be an int - with self.assertRaises(TypeError): - cycle('defg').__setstate__((list('abcdefg'), 'x')) - - self.assertRaises(TypeError, cycle('').__setstate__, ()) - self.assertRaises(TypeError, cycle('').__setstate__, ([],)) - - @pickle_deprecated def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) @@ -831,15 +615,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check normal pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for elem in g: - self.assertEqual(k, elem[0]) - dup.append(elem) - self.assertEqual(s, dup) - # Check nested case dup = [] for k, g in groupby(s, testR): @@ -850,18 +625,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check nested and pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for ik, ig in pickle.loads(pickle.dumps(groupby(g, testR2), proto)): - for elem in ig: - self.assertEqual(k, elem[0]) - self.assertEqual(ik, elem[2]) - dup.append(elem) - self.assertEqual(s, dup) - - # Check case where inner iterator is not used keys = [k for k, g in groupby(s, testR)] expectedkeys = set([r[0] for r in s]) @@ -881,13 +644,6 @@ def test_groupby(self): list(it) # exhaust the groupby iterator self.assertEqual(list(g3), []) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = groupby(s, testR) - _, g = next(it) - next(it) - next(it) - self.assertEqual(list(pickle.loads(pickle.dumps(g, proto))), []) - # Exercise pipes and filters style s = 'abracadabra' # sort s | uniq @@ -970,7 +726,6 @@ def test_filter(self): c = filter(isEven, range(6)) self.pickletest(proto, c) - @pickle_deprecated def test_filterfalse(self): self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) self.assertEqual(list(filterfalse(None, [0,1,0,2,0])), [0,0,0]) @@ -981,8 +736,6 @@ def test_filterfalse(self): self.assertRaises(TypeError, filterfalse, lambda x:x, range(6), 7) self.assertRaises(TypeError, filterfalse, isEven, 3) self.assertRaises(TypeError, next, filterfalse(range(6), range(6))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, filterfalse(isEven, range(6))) def test_zip(self): # XXX This is rather silly now that builtin zip() calls zip()... @@ -1001,33 +754,12 @@ def test_zip(self): lzip('abc', 'def')) @support.impl_detail("tuple reuse is specific to CPython") - @pickle_deprecated def test_zip_tuple_reuse(self): ids = list(map(id, zip('abc', 'def'))) self.assertEqual(min(ids), max(ids)) ids = list(map(id, list(zip('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - # check copy, deepcopy, pickle - ans = [(x,y) for x, y in copy.copy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - ans = [(x,y) for x, y in copy.deepcopy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(zip('abc',count()), proto))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - testIntermediate = zip('abc',count()) - next(testIntermediate) - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(testIntermediate, proto))] - self.assertEqual(ans, [('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip('abc', count())) - def test_ziplongest(self): for args in [ ['abc', range(6)], @@ -1077,14 +809,6 @@ def test_zip_longest_tuple_reuse(self): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - @pickle_deprecated - def test_zip_longest_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip_longest("abc", "def")) - self.pickletest(proto, zip_longest("abc", "defgh")) - self.pickletest(proto, zip_longest("abc", "defgh", fillvalue=1)) - self.pickletest(proto, zip_longest("", "defgh")) - def test_zip_longest_bad_iterable(self): exception = TypeError() @@ -1296,34 +1020,6 @@ def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) - @pickle_deprecated - def test_product_pickling(self): - # check copy, deepcopy, pickle - for args, result in [ - ([], [()]), # zero iterables - (['ab'], [('a',), ('b',)]), # one iterable - ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables - ([range(0), range(2), range(3)], []), # first iterable with zero length - ([range(2), range(0), range(3)], []), # middle iterable with zero length - ([range(2), range(3), range(0)], []), # last iterable with zero length - ]: - self.assertEqual(list(copy.copy(product(*args))), result) - self.assertEqual(list(copy.deepcopy(product(*args))), result) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, product(*args)) - - @pickle_deprecated - def test_product_issue_25021(self): - # test that indices are properly clamped to the length of the tuples - p = product((1, 2),(3,)) - p.__setstate__((0, 0x1000)) # will access tuple element 1 if not clamped - self.assertEqual(next(p), (2, 3)) - # test that empty tuple in the list will result in an immediate StopIteration - p = product((1, 2), (), (3,)) - p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped - self.assertRaises(StopIteration, next, p) - - @pickle_deprecated def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) self.assertEqual(lzip(range(3),repeat('a')), @@ -1342,21 +1038,12 @@ def test_repeat(self): list(r) self.assertEqual(repr(r), 'repeat((1+0j), 0)') - # check copy, deepcopy, pickle - c = repeat(object='a', times=10) - self.assertEqual(next(c), 'a') - self.assertEqual(take(2, copy.copy(c)), list('a' * 2)) - self.assertEqual(take(2, copy.deepcopy(c)), list('a' * 2)) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, repeat(object='a', times=10)) - def test_repeat_with_negative_times(self): self.assertEqual(repr(repeat('a', -1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', -2)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") - @pickle_deprecated def test_map(self): self.assertEqual(list(map(operator.pow, range(3), range(1,7))), [0**1, 1**2, 2**3]) @@ -1374,20 +1061,6 @@ def test_map(self): self.assertRaises(ValueError, next, map(errfunc, [4], [5])) self.assertRaises(TypeError, next, map(onearg, [4], [5])) - # check copy, deepcopy, pickle - ans = [('a',0),('b',1),('c',2)] - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.copy(c)), ans) - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = map(tupleize, 'abc', count()) - self.pickletest(proto, c) - - @pickle_deprecated def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), [0**1, 1**2, 2**3]) @@ -1402,20 +1075,6 @@ def test_starmap(self): self.assertRaises(ValueError, next, starmap(errfunc, [(4,5)])) self.assertRaises(TypeError, next, starmap(onearg, [(4,5)])) - # check copy, deepcopy, pickle - ans = [0**1, 1**2, 2**3] - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.copy(c)), ans) - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.pickletest(proto, c) - - @pickle_deprecated def test_islice(self): for args in [ # islice(args) should agree with range(args) (10, 20, 3), @@ -1472,21 +1131,6 @@ def test_islice(self): self.assertEqual(list(islice(c, 1, 3, 50)), [1]) self.assertEqual(next(c), 3) - # check copy, deepcopy, pickle - for args in [ # islice(args) should agree with range(args) - (10, 20, 3), - (10, 3, 20), - (10, 20), - (10, 3), - (20,) - ]: - self.assertEqual(list(copy.copy(islice(range(100), *args))), - list(range(*args))) - self.assertEqual(list(copy.deepcopy(islice(range(100), *args))), - list(range(*args))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, islice(range(100), *args)) - # Issue #21321: check source iterator is not referenced # from islice() after the latter has been exhausted it = (x for x in (1, 2)) @@ -1510,7 +1154,6 @@ def __index__(self): self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), list(range(10,50,5))) - @pickle_deprecated def test_takewhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(takewhile(underten, data)), [1, 3, 5]) @@ -1524,14 +1167,6 @@ def test_takewhile(self): self.assertEqual(list(t), [1, 1, 1]) self.assertRaises(StopIteration, next, t) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(takewhile(underten, data))), [1, 3, 5]) - self.assertEqual(list(copy.deepcopy(takewhile(underten, data))), - [1, 3, 5]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, takewhile(underten, data)) - - @pickle_deprecated def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) @@ -1542,14 +1177,6 @@ def test_dropwhile(self): self.assertRaises(TypeError, next, dropwhile(10, [(4,5)])) self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)])) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(dropwhile(underten, data))), [20, 2, 4, 6, 8]) - self.assertEqual(list(copy.deepcopy(dropwhile(underten, data))), - [20, 2, 4, 6, 8]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, dropwhile(underten, data)) - - @pickle_deprecated def test_tee(self): n = 200 @@ -1664,41 +1291,6 @@ def test_tee(self): self.assertEqual(list(a), long_ans[100:]) self.assertEqual(list(b), long_ans[60:]) - # check deepcopy - a, b = tee('abc') - self.assertEqual(list(copy.deepcopy(a)), ans) - self.assertEqual(list(copy.deepcopy(b)), ans) - self.assertEqual(list(a), ans) - self.assertEqual(list(b), ans) - a, b = tee(range(10000)) - self.assertEqual(list(copy.deepcopy(a)), long_ans) - self.assertEqual(list(copy.deepcopy(b)), long_ans) - self.assertEqual(list(a), long_ans) - self.assertEqual(list(b), long_ans) - - # check partially consumed deepcopy - a, b = tee('abc') - take(2, a) - take(1, b) - self.assertEqual(list(copy.deepcopy(a)), ans[2:]) - self.assertEqual(list(copy.deepcopy(b)), ans[1:]) - self.assertEqual(list(a), ans[2:]) - self.assertEqual(list(b), ans[1:]) - a, b = tee(range(10000)) - take(100, a) - take(60, b) - self.assertEqual(list(copy.deepcopy(a)), long_ans[100:]) - self.assertEqual(list(copy.deepcopy(b)), long_ans[60:]) - self.assertEqual(list(a), long_ans[100:]) - self.assertEqual(list(b), long_ans[60:]) - - # check pickle - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, iter(tee('abc'))) - a, b = tee('abc') - self.pickletest(proto, a, compare=ans) - self.pickletest(proto, b, compare=ans) - def test_tee_dealloc_segfault(self): # gh-115874: segfaults when accessing module state in tp_dealloc. script = ( @@ -1866,33 +1458,6 @@ class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) - @pickle_deprecated - def test_accumulate_reducible(self): - # check copy, deepcopy, pickle - data = [1, 2, 3, 4, 5] - accumulated = [1, 3, 6, 10, 15] - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = accumulate(data) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[:]) - self.assertEqual(next(it), 1) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[1:]) - it = accumulate(data) - self.assertEqual(next(it), 1) - self.assertEqual(list(copy.deepcopy(it)), accumulated[1:]) - self.assertEqual(list(copy.copy(it)), accumulated[1:]) - - @pickle_deprecated - def test_accumulate_reducible_none(self): - # Issue #25718: total is None - it = accumulate([None, None, None], operator.is_) - self.assertEqual(next(it), None) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it_copy = pickle.loads(pickle.dumps(it, proto)) - self.assertEqual(list(it_copy), [True, False]) - self.assertEqual(list(copy.deepcopy(it)), [True, False]) - self.assertEqual(list(copy.copy(it)), [True, False]) - def test_chain(self): self.assertEqual(''.join(chain('ABC', 'DEF')), 'ABCDEF') diff --git a/Misc/NEWS.d/next/Library/2024-05-09-02-43-37.gh-issue-101588.30bNAr.rst b/Misc/NEWS.d/next/Library/2024-05-09-02-43-37.gh-issue-101588.30bNAr.rst new file mode 100644 index 00000000000000..3e0f496047bc8e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-02-43-37.gh-issue-101588.30bNAr.rst @@ -0,0 +1,2 @@ +Remove copy, deepcopy, and pickle from itertools. These had previously +raised a DeprecationWarning since Python 3.12. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 8641c2f87e6db2..ae316d9e369d45 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -93,15 +93,6 @@ class itertools.pairwise "pairwiseobject *" "clinic_state()->pairwise_type" #undef clinic_state_by_cls #undef clinic_state -/* Deprecation of pickle support: GH-101588 *********************************/ - -#define ITERTOOL_PICKLE_DEPRECATION \ - if (PyErr_WarnEx( \ - PyExc_DeprecationWarning, \ - "Pickle, copy, and deepcopy support will be " \ - "removed from itertools in Python 3.14.", 1) < 0) { \ - return NULL; \ - } /* batched object ************************************************************/ @@ -554,57 +545,6 @@ groupby_next(groupbyobject *gbo) return r; } -static PyObject * -groupby_reduce(groupbyobject *lz, PyObject *Py_UNUSED(ignored)) -{ - /* reduce as a 'new' call with an optional 'setstate' if groupby - * has started - */ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *value; - if (lz->tgtkey && lz->currkey && lz->currvalue) - value = Py_BuildValue("O(OO)(OOO)", Py_TYPE(lz), - lz->it, lz->keyfunc, lz->currkey, lz->currvalue, lz->tgtkey); - else - value = Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->it, lz->keyfunc); - - return value; -} - -PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); - -static PyObject * -groupby_setstate(groupbyobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *currkey, *currvalue, *tgtkey; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "OOO", &currkey, &currvalue, &tgtkey)) { - return NULL; - } - Py_INCREF(currkey); - Py_XSETREF(lz->currkey, currkey); - Py_INCREF(currvalue); - Py_XSETREF(lz->currvalue, currvalue); - Py_INCREF(tgtkey); - Py_XSETREF(lz->tgtkey, tgtkey); - Py_RETURN_NONE; -} - -PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); - -static PyMethodDef groupby_methods[] = { - {"__reduce__", (PyCFunction)groupby_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)groupby_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot groupby_slots[] = { {Py_tp_dealloc, groupby_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -612,7 +552,6 @@ static PyType_Slot groupby_slots[] = { {Py_tp_traverse, groupby_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, groupby_next}, - {Py_tp_methods, groupby_methods}, {Py_tp_new, itertools_groupby}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -713,29 +652,12 @@ _grouper_next(_grouperobject *igo) return r; } -static PyObject * -_grouper_reduce(_grouperobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (((groupbyobject *)lz->parent)->currgrouper != lz) { - return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter))); - } - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->parent, lz->tgtkey); -} - -static PyMethodDef _grouper_methods[] = { - {"__reduce__", (PyCFunction)_grouper_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot _grouper_slots[] = { {Py_tp_dealloc, _grouper_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_traverse, _grouper_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, _grouper_next}, - {Py_tp_methods, _grouper_methods}, {Py_tp_new, itertools__grouper}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -880,25 +802,6 @@ teedataobject_dealloc(teedataobject *tdo) Py_DECREF(tp); } -static PyObject * -teedataobject_reduce(teedataobject *tdo, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - int i; - /* create a temporary list of already iterated values */ - PyObject *values = PyList_New(tdo->numread); - - if (!values) - return NULL; - for (i=0 ; inumread ; i++) { - Py_INCREF(tdo->values[i]); - PyList_SET_ITEM(values, i, tdo->values[i]); - } - return Py_BuildValue("O(ONO)", Py_TYPE(tdo), tdo->it, - values, - tdo->nextlink ? tdo->nextlink : Py_None); -} - /*[clinic input] @classmethod itertools.teedataobject.__new__ @@ -953,19 +856,12 @@ itertools_teedataobject_impl(PyTypeObject *type, PyObject *it, return NULL; } -static PyMethodDef teedataobject_methods[] = { - {"__reduce__", (PyCFunction)teedataobject_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot teedataobject_slots[] = { {Py_tp_dealloc, teedataobject_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_doc, (void *)itertools_teedataobject__doc__}, {Py_tp_traverse, teedataobject_traverse}, {Py_tp_clear, teedataobject_clear}, - {Py_tp_methods, teedataobject_methods}, {Py_tp_new, itertools_teedataobject}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1094,41 +990,8 @@ tee_dealloc(teeobject *to) Py_DECREF(tp); } -static PyObject * -tee_reduce(teeobject *to, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(())(Oi)", Py_TYPE(to), to->dataobj, to->index); -} - -static PyObject * -tee_setstate(teeobject *to, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - teedataobject *tdo; - int index; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - PyTypeObject *tdo_type = to->state->teedataobject_type; - if (!PyArg_ParseTuple(state, "O!i", tdo_type, &tdo, &index)) { - return NULL; - } - if (index < 0 || index > LINKCELLS) { - PyErr_SetString(PyExc_ValueError, "Index out of range"); - return NULL; - } - Py_INCREF(tdo); - Py_XSETREF(to->dataobj, tdo); - to->index = index; - Py_RETURN_NONE; -} - static PyMethodDef tee_methods[] = { {"__copy__", (PyCFunction)tee_copy, METH_NOARGS, teecopy_doc}, - {"__reduce__", (PyCFunction)tee_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)tee_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -1330,59 +1193,6 @@ cycle_next(cycleobject *lz) return Py_NewRef(item); } -static PyObject * -cycle_reduce(cycleobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Create a new cycle with the iterator tuple, then set the saved state */ - if (lz->it == NULL) { - PyObject *it = PyObject_GetIter(lz->saved); - if (it == NULL) - return NULL; - if (lz->index != 0) { - PyObject *res = _PyObject_CallMethod(it, &_Py_ID(__setstate__), - "n", lz->index); - if (res == NULL) { - Py_DECREF(it); - return NULL; - } - Py_DECREF(res); - } - return Py_BuildValue("O(N)(OO)", Py_TYPE(lz), it, lz->saved, Py_True); - } - return Py_BuildValue("O(O)(OO)", Py_TYPE(lz), lz->it, lz->saved, - lz->firstpass ? Py_True : Py_False); -} - -static PyObject * -cycle_setstate(cycleobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *saved=NULL; - int firstpass; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - // The second item can be 1/0 in old pickles and True/False in new pickles - if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { - return NULL; - } - Py_INCREF(saved); - Py_XSETREF(lz->saved, saved); - lz->firstpass = firstpass != 0; - lz->index = 0; - Py_RETURN_NONE; -} - -static PyMethodDef cycle_methods[] = { - {"__reduce__", (PyCFunction)cycle_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)cycle_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot cycle_slots[] = { {Py_tp_dealloc, cycle_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1390,7 +1200,6 @@ static PyType_Slot cycle_slots[] = { {Py_tp_traverse, cycle_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, cycle_next}, - {Py_tp_methods, cycle_methods}, {Py_tp_new, itertools_cycle}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1503,32 +1312,6 @@ dropwhile_next(dropwhileobject *lz) } } -static PyObject * -dropwhile_reduce(dropwhileobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->start); -} - -static PyObject * -dropwhile_setstate(dropwhileobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - int start = PyObject_IsTrue(state); - if (start < 0) - return NULL; - lz->start = start; - Py_RETURN_NONE; -} - -static PyMethodDef dropwhile_methods[] = { - {"__reduce__", (PyCFunction)dropwhile_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)dropwhile_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot dropwhile_slots[] = { {Py_tp_dealloc, dropwhile_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1536,7 +1319,6 @@ static PyType_Slot dropwhile_slots[] = { {Py_tp_traverse, dropwhile_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, dropwhile_next}, - {Py_tp_methods, dropwhile_methods}, {Py_tp_new, itertools_dropwhile}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1643,33 +1425,6 @@ takewhile_next(takewhileobject *lz) return NULL; } -static PyObject * -takewhile_reduce(takewhileobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->stop); -} - -static PyObject * -takewhile_reduce_setstate(takewhileobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - int stop = PyObject_IsTrue(state); - - if (stop < 0) - return NULL; - lz->stop = stop; - Py_RETURN_NONE; -} - -static PyMethodDef takewhile_reduce_methods[] = { - {"__reduce__", (PyCFunction)takewhile_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)takewhile_reduce_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot takewhile_slots[] = { {Py_tp_dealloc, takewhile_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1677,7 +1432,6 @@ static PyType_Slot takewhile_slots[] = { {Py_tp_traverse, takewhile_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, takewhile_next}, - {Py_tp_methods, takewhile_reduce_methods}, {Py_tp_new, itertools_takewhile}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1847,59 +1601,6 @@ islice_next(isliceobject *lz) return NULL; } -static PyObject * -islice_reduce(isliceobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* When unpickled, generate a new object with the same bounds, - * then 'setstate' with the next and count - */ - PyObject *stop; - - if (lz->it == NULL) { - PyObject *empty_list; - PyObject *empty_it; - empty_list = PyList_New(0); - if (empty_list == NULL) - return NULL; - empty_it = PyObject_GetIter(empty_list); - Py_DECREF(empty_list); - if (empty_it == NULL) - return NULL; - return Py_BuildValue("O(Nn)n", Py_TYPE(lz), empty_it, 0, 0); - } - if (lz->stop == -1) { - stop = Py_NewRef(Py_None); - } else { - stop = PyLong_FromSsize_t(lz->stop); - if (stop == NULL) - return NULL; - } - return Py_BuildValue("O(OnNn)n", Py_TYPE(lz), - lz->it, lz->next, stop, lz->step, - lz->cnt); -} - -static PyObject * -islice_setstate(isliceobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_ssize_t cnt = PyLong_AsSsize_t(state); - - if (cnt == -1 && PyErr_Occurred()) - return NULL; - lz->cnt = cnt; - Py_RETURN_NONE; -} - -static PyMethodDef islice_methods[] = { - {"__reduce__", (PyCFunction)islice_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)islice_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - PyDoc_STRVAR(islice_doc, "islice(iterable, stop) --> islice object\n\ islice(iterable, start, stop[, step]) --> islice object\n\ @@ -1918,7 +1619,6 @@ static PyType_Slot islice_slots[] = { {Py_tp_traverse, islice_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, islice_next}, - {Py_tp_methods, islice_methods}, {Py_tp_new, islice_new}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -2016,20 +1716,6 @@ starmap_next(starmapobject *lz) return result; } -static PyObject * -starmap_reduce(starmapobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Just pickle the iterator */ - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->func, lz->it); -} - -static PyMethodDef starmap_methods[] = { - {"__reduce__", (PyCFunction)starmap_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot starmap_slots[] = { {Py_tp_dealloc, starmap_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -2037,7 +1723,6 @@ static PyType_Slot starmap_slots[] = { {Py_tp_traverse, starmap_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, starmap_next}, - {Py_tp_methods, starmap_methods}, {Py_tp_new, itertools_starmap}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -2173,51 +1858,6 @@ chain_next(chainobject *lz) return NULL; } -static PyObject * -chain_reduce(chainobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->source) { - /* we can't pickle function objects (itertools.from_iterable) so - * we must use setstate to replace the iterable. One day we - * will fix pickling of functions - */ - if (lz->active) { - return Py_BuildValue("O()(OO)", Py_TYPE(lz), lz->source, lz->active); - } else { - return Py_BuildValue("O()(O)", Py_TYPE(lz), lz->source); - } - } else { - return Py_BuildValue("O()", Py_TYPE(lz)); /* exhausted */ - } - return NULL; -} - -static PyObject * -chain_setstate(chainobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *source, *active=NULL; - - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "O|O", &source, &active)) { - return NULL; - } - if (!PyIter_Check(source) || (active != NULL && !PyIter_Check(active))) { - PyErr_SetString(PyExc_TypeError, "Arguments must be iterators."); - return NULL; - } - - Py_INCREF(source); - Py_XSETREF(lz->source, source); - Py_XINCREF(active); - Py_XSETREF(lz->active, active); - Py_RETURN_NONE; -} - PyDoc_STRVAR(chain_doc, "chain(*iterables)\n\ --\n\ @@ -2228,10 +1868,6 @@ iterable, until all of the iterables are exhausted."); static PyMethodDef chain_methods[] = { ITERTOOLS_CHAIN_FROM_ITERABLE_METHODDEF - {"__reduce__", (PyCFunction)chain_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)chain_setstate, METH_O, - setstate_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -2470,89 +2106,7 @@ product_next(productobject *lz) return NULL; } -static PyObject * -product_reduce(productobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->stopped) { - return Py_BuildValue("O(())", Py_TYPE(lz)); - } else if (lz->result == NULL) { - return Py_BuildValue("OO", Py_TYPE(lz), lz->pools); - } else { - PyObject *indices; - Py_ssize_t n, i; - - /* we must pickle the indices use them for setstate, and - * additionally indicate that the iterator has started - */ - n = PyTuple_GET_SIZE(lz->pools); - indices = PyTuple_New(n); - if (indices == NULL) - return NULL; - for (i=0; iindices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - return Py_BuildValue("OON", Py_TYPE(lz), lz->pools, indices); - } -} - -static PyObject * -product_setstate(productobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t n, i; - - n = PyTuple_GET_SIZE(lz->pools); - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != n) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - for (i=0; ipools, i); - poolsize = PyTuple_GET_SIZE(pool); - if (poolsize == 0) { - lz->stopped = 1; - Py_RETURN_NONE; - } - /* clamp the index */ - if (index < 0) - index = 0; - else if (index > poolsize-1) - index = poolsize-1; - lz->indices[i] = index; - } - - result = PyTuple_New(n); - if (!result) - return NULL; - for (i=0; ipools, i); - PyObject *element = PyTuple_GET_ITEM(pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef product_methods[] = { - {"__reduce__", (PyCFunction)product_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)product_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)product_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -2781,83 +2335,7 @@ combinations_next(combinationsobject *co) return NULL; } -static PyObject * -combinations_reduce(combinationsobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(lz), lz->pool, lz->r); - } else if (lz->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(lz), lz->r); - } else { - PyObject *indices; - Py_ssize_t i; - - /* we must pickle the indices and use them for setstate */ - indices = PyTuple_New(lz->r); - if (!indices) - return NULL; - for (i=0; ir; i++) - { - PyObject* index = PyLong_FromSsize_t(lz->indices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - - return Py_BuildValue("O(On)N", Py_TYPE(lz), lz->pool, lz->r, indices); - } -} - -static PyObject * -combinations_setstate(combinationsobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t i; - Py_ssize_t n = PyTuple_GET_SIZE(lz->pool); - - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - for (i=0; ir; i++) { - Py_ssize_t max; - PyObject* indexObject = PyTuple_GET_ITEM(state, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - - if (index == -1 && PyErr_Occurred()) - return NULL; /* not an integer */ - max = i + n - lz->r; - /* clamp the index (beware of negative max) */ - if (index > max) - index = max; - if (index < 0) - index = 0; - lz->indices[i] = index; - } - - result = PyTuple_New(lz->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(lz->pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef combinations_methods[] = { - {"__reduce__", (PyCFunction)combinations_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)combinations_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)combinations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3091,79 +2569,7 @@ cwr_next(cwrobject *co) return NULL; } -static PyObject * -cwr_reduce(cwrobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(lz), lz->pool, lz->r); - } else if (lz->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(lz), lz->r); - } else { - PyObject *indices; - Py_ssize_t i; - - /* we must pickle the indices and use them for setstate */ - indices = PyTuple_New(lz->r); - if (!indices) - return NULL; - for (i=0; ir; i++) { - PyObject* index = PyLong_FromSsize_t(lz->indices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - - return Py_BuildValue("O(On)N", Py_TYPE(lz), lz->pool, lz->r, indices); - } -} - -static PyObject * -cwr_setstate(cwrobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t n, i; - - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) - { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - n = PyTuple_GET_SIZE(lz->pool); - for (i=0; ir; i++) { - PyObject* indexObject = PyTuple_GET_ITEM(state, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - - if (index < 0 && PyErr_Occurred()) - return NULL; /* not an integer */ - /* clamp the index */ - if (index < 0) - index = 0; - else if (index > n-1) - index = n-1; - lz->indices[i] = index; - } - result = PyTuple_New(lz->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(lz->pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef cwr_methods[] = { - {"__reduce__", (PyCFunction)cwr_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)cwr_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)cwr_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3428,113 +2834,7 @@ permutations_next(permutationsobject *po) return NULL; } -static PyObject * -permutations_reduce(permutationsobject *po, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (po->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(po), po->pool, po->r); - } else if (po->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(po), po->r); - } else { - PyObject *indices=NULL, *cycles=NULL; - Py_ssize_t n, i; - - /* we must pickle the indices and cycles and use them for setstate */ - n = PyTuple_GET_SIZE(po->pool); - indices = PyTuple_New(n); - if (indices == NULL) - goto err; - for (i=0; iindices[i]); - if (!index) - goto err; - PyTuple_SET_ITEM(indices, i, index); - } - - cycles = PyTuple_New(po->r); - if (cycles == NULL) - goto err; - for (i=0 ; ir ; i++) { - PyObject* index = PyLong_FromSsize_t(po->cycles[i]); - if (!index) - goto err; - PyTuple_SET_ITEM(cycles, i, index); - } - return Py_BuildValue("O(On)(NN)", Py_TYPE(po), - po->pool, po->r, - indices, cycles); - err: - Py_XDECREF(indices); - Py_XDECREF(cycles); - return NULL; - } -} - -static PyObject * -permutations_setstate(permutationsobject *po, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *indices, *cycles, *result; - Py_ssize_t n, i; - - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "O!O!", - &PyTuple_Type, &indices, - &PyTuple_Type, &cycles)) { - return NULL; - } - - n = PyTuple_GET_SIZE(po->pool); - if (PyTuple_GET_SIZE(indices) != n || PyTuple_GET_SIZE(cycles) != po->r) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - for (i=0; i n-1) - index = n-1; - po->indices[i] = index; - } - - for (i=0; ir; i++) { - PyObject* indexObject = PyTuple_GET_ITEM(cycles, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - if (index < 0 && PyErr_Occurred()) - return NULL; /* not an integer */ - if (index < 1) - index = 1; - else if (index > n-i) - index = n-i; - po->cycles[i] = index; - } - result = PyTuple_New(po->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(po->pool, po->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(po->result, result); - Py_RETURN_NONE; -} - static PyMethodDef permuations_methods[] = { - {"__reduce__", (PyCFunction)permutations_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)permutations_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)permutations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3669,59 +2969,6 @@ accumulate_next(accumulateobject *lz) return newtotal; } -static PyObject * -accumulate_reduce(accumulateobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - itertools_state *state = lz->state; - - if (lz->initial != Py_None) { - PyObject *it; - - assert(lz->total == NULL); - it = PyObject_CallFunction((PyObject *)(state->chain_type), "(O)O", - lz->initial, lz->it); - if (it == NULL) - return NULL; - return Py_BuildValue("O(NO)O", Py_TYPE(lz), - it, lz->binop?lz->binop:Py_None, Py_None); - } - if (lz->total == Py_None) { - PyObject *it; - - it = PyObject_CallFunction((PyObject *)(state->chain_type), "(O)O", - lz->total, lz->it); - if (it == NULL) - return NULL; - it = PyObject_CallFunction((PyObject *)Py_TYPE(lz), "NO", - it, lz->binop ? lz->binop : Py_None); - if (it == NULL) - return NULL; - - return Py_BuildValue("O(NiO)", state->islice_type, it, 1, Py_None); - } - return Py_BuildValue("O(OO)O", Py_TYPE(lz), - lz->it, lz->binop?lz->binop:Py_None, - lz->total?lz->total:Py_None); -} - -static PyObject * -accumulate_setstate(accumulateobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_INCREF(state); - Py_XSETREF(lz->total, state); - Py_RETURN_NONE; -} - -static PyMethodDef accumulate_methods[] = { - {"__reduce__", (PyCFunction)accumulate_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)accumulate_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot accumulate_slots[] = { {Py_tp_dealloc, accumulate_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3729,7 +2976,6 @@ static PyType_Slot accumulate_slots[] = { {Py_tp_traverse, accumulate_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, accumulate_next}, - {Py_tp_methods, accumulate_methods}, {Py_tp_new, itertools_accumulate}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -3854,20 +3100,6 @@ compress_next(compressobject *lz) } } -static PyObject * -compress_reduce(compressobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->data, lz->selectors); -} - -static PyMethodDef compress_methods[] = { - {"__reduce__", (PyCFunction)compress_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot compress_slots[] = { {Py_tp_dealloc, compress_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3875,7 +3107,6 @@ static PyType_Slot compress_slots[] = { {Py_tp_traverse, compress_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, compress_next}, - {Py_tp_methods, compress_methods}, {Py_tp_new, itertools_compress}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -3987,19 +3218,6 @@ filterfalse_next(filterfalseobject *lz) } } -static PyObject * -filterfalse_reduce(filterfalseobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->func, lz->it); -} - -static PyMethodDef filterfalse_methods[] = { - {"__reduce__", (PyCFunction)filterfalse_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot filterfalse_slots[] = { {Py_tp_dealloc, filterfalse_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -4007,7 +3225,6 @@ static PyType_Slot filterfalse_slots[] = { {Py_tp_traverse, filterfalse_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, filterfalse_next}, - {Py_tp_methods, filterfalse_methods}, {Py_tp_new, itertools_filterfalse}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -4215,21 +3432,6 @@ count_repr(countobject *lz) lz->long_cnt, lz->long_step); } -static PyObject * -count_reduce(countobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->cnt == PY_SSIZE_T_MAX) - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->long_cnt, lz->long_step); - return Py_BuildValue("O(n)", Py_TYPE(lz), lz->cnt); -} - -static PyMethodDef count_methods[] = { - {"__reduce__", (PyCFunction)count_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot count_slots[] = { {Py_tp_dealloc, count_dealloc}, {Py_tp_repr, count_repr}, @@ -4238,7 +3440,6 @@ static PyType_Slot count_slots[] = { {Py_tp_traverse, count_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, count_next}, - {Py_tp_methods, count_methods}, {Py_tp_new, itertools_count}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -4339,22 +3540,8 @@ repeat_len(repeatobject *ro, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); -static PyObject * -repeat_reduce(repeatobject *ro, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* unpickle this so that a new repeat iterator is constructed with an - * object, then call __setstate__ on it to set cnt - */ - if (ro->cnt >= 0) - return Py_BuildValue("O(On)", Py_TYPE(ro), ro->element, ro->cnt); - else - return Py_BuildValue("O(O)", Py_TYPE(ro), ro->element); -} - static PyMethodDef repeat_methods[] = { {"__length_hint__", (PyCFunction)repeat_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)repeat_reduce, METH_NOARGS, reduce_doc}, {NULL, NULL} /* sentinel */ }; @@ -4560,50 +3747,6 @@ zip_longest_next(ziplongestobject *lz) return result; } -static PyObject * -zip_longest_reduce(ziplongestobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Create a new tuple with empty sequences where appropriate to pickle. - * Then use setstate to set the fillvalue - */ - int i; - PyObject *args = PyTuple_New(PyTuple_GET_SIZE(lz->ittuple)); - - if (args == NULL) - return NULL; - for (i=0; iittuple); i++) { - PyObject *elem = PyTuple_GET_ITEM(lz->ittuple, i); - if (elem == NULL) { - elem = PyTuple_New(0); - if (elem == NULL) { - Py_DECREF(args); - return NULL; - } - } else - Py_INCREF(elem); - PyTuple_SET_ITEM(args, i, elem); - } - return Py_BuildValue("ONO", Py_TYPE(lz), args, lz->fillvalue); -} - -static PyObject * -zip_longest_setstate(ziplongestobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_INCREF(state); - Py_XSETREF(lz->fillvalue, state); - Py_RETURN_NONE; -} - -static PyMethodDef zip_longest_methods[] = { - {"__reduce__", (PyCFunction)zip_longest_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)zip_longest_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - PyDoc_STRVAR(zip_longest_doc, "zip_longest(*iterables, fillvalue=None)\n\ --\n\ @@ -4623,7 +3766,6 @@ static PyType_Slot ziplongest_slots[] = { {Py_tp_traverse, zip_longest_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, zip_longest_next}, - {Py_tp_methods, zip_longest_methods}, {Py_tp_new, zip_longest_new}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, From fa9b9cb11379806843ae03b1e4ad4ccd95a63c02 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 9 May 2024 11:36:17 +0300 Subject: [PATCH 017/903] gh-118033: Fix `__weakref__` not set for generic dataclasses (#118099) --- Lib/dataclasses.py | 13 ++- Lib/test/test_dataclasses/__init__.py | 106 ++++++++++++++++++ ...-04-19-14-59-53.gh-issue-118033.amS4Gw.rst | 2 + 3 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-19-14-59-53.gh-issue-118033.amS4Gw.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3acd03cd865234..aeafbfbbe6e9c4 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1199,10 +1199,17 @@ def _dataclass_setstate(self, state): def _get_slots(cls): match cls.__dict__.get('__slots__'): - # A class which does not define __slots__ at all is equivalent - # to a class defining __slots__ = ('__dict__', '__weakref__') + # `__dictoffset__` and `__weakrefoffset__` can tell us whether + # the base type has dict/weakref slots, in a way that works correctly + # for both Python classes and C extension types. Extension types + # don't use `__slots__` for slot creation case None: - yield from ('__dict__', '__weakref__') + slots = [] + if getattr(cls, '__weakrefoffset__', -1) != 0: + slots.append('__weakref__') + if getattr(cls, '__dictrefoffset__', -1) != 0: + slots.append('__dict__') + yield from slots case str(slot): yield slot # Slots may be any iterable, but we cannot handle an iterator diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 832e5672c77d0d..ea49596eaa4d96 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -3515,8 +3515,114 @@ class A: class B(A): pass + self.assertEqual(B.__slots__, ()) B() + def test_dataclass_derived_generic(self): + T = typing.TypeVar('T') + + @dataclass(slots=True, weakref_slot=True) + class A(typing.Generic[T]): + pass + self.assertEqual(A.__slots__, ('__weakref__',)) + self.assertTrue(A.__weakref__) + A() + + @dataclass(slots=True, weakref_slot=True) + class B[T2]: + pass + self.assertEqual(B.__slots__, ('__weakref__',)) + self.assertTrue(B.__weakref__) + B() + + def test_dataclass_derived_generic_from_base(self): + T = typing.TypeVar('T') + + class RawBase: ... + + @dataclass(slots=True, weakref_slot=True) + class C1(typing.Generic[T], RawBase): + pass + self.assertEqual(C1.__slots__, ()) + self.assertTrue(C1.__weakref__) + C1() + @dataclass(slots=True, weakref_slot=True) + class C2(RawBase, typing.Generic[T]): + pass + self.assertEqual(C2.__slots__, ()) + self.assertTrue(C2.__weakref__) + C2() + + @dataclass(slots=True, weakref_slot=True) + class D[T2](RawBase): + pass + self.assertEqual(D.__slots__, ()) + self.assertTrue(D.__weakref__) + D() + + def test_dataclass_derived_generic_from_slotted_base(self): + T = typing.TypeVar('T') + + class WithSlots: + __slots__ = ('a', 'b') + + @dataclass(slots=True, weakref_slot=True) + class E1(WithSlots, Generic[T]): + pass + self.assertEqual(E1.__slots__, ('__weakref__',)) + self.assertTrue(E1.__weakref__) + E1() + @dataclass(slots=True, weakref_slot=True) + class E2(Generic[T], WithSlots): + pass + self.assertEqual(E2.__slots__, ('__weakref__',)) + self.assertTrue(E2.__weakref__) + E2() + + @dataclass(slots=True, weakref_slot=True) + class F[T2](WithSlots): + pass + self.assertEqual(F.__slots__, ('__weakref__',)) + self.assertTrue(F.__weakref__) + F() + + def test_dataclass_derived_generic_from_slotted_base(self): + T = typing.TypeVar('T') + + class WithWeakrefSlot: + __slots__ = ('__weakref__',) + + @dataclass(slots=True, weakref_slot=True) + class G1(WithWeakrefSlot, Generic[T]): + pass + self.assertEqual(G1.__slots__, ()) + self.assertTrue(G1.__weakref__) + G1() + @dataclass(slots=True, weakref_slot=True) + class G2(Generic[T], WithWeakrefSlot): + pass + self.assertEqual(G2.__slots__, ()) + self.assertTrue(G2.__weakref__) + G2() + + @dataclass(slots=True, weakref_slot=True) + class H[T2](WithWeakrefSlot): + pass + self.assertEqual(H.__slots__, ()) + self.assertTrue(H.__weakref__) + H() + + def test_dataclass_slot_dict(self): + class WithDictSlot: + __slots__ = ('__dict__',) + + @dataclass(slots=True) + class A(WithDictSlot): ... + + self.assertEqual(A.__slots__, ()) + self.assertEqual(A().__dict__, {}) + A() + class TestDescriptors(unittest.TestCase): def test_set_name(self): diff --git a/Misc/NEWS.d/next/Library/2024-04-19-14-59-53.gh-issue-118033.amS4Gw.rst b/Misc/NEWS.d/next/Library/2024-04-19-14-59-53.gh-issue-118033.amS4Gw.rst new file mode 100644 index 00000000000000..7ceb29330abf22 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-19-14-59-53.gh-issue-118033.amS4Gw.rst @@ -0,0 +1,2 @@ +Fix :func:`dataclasses.dataclass` not creating a ``__weakref__`` slot when +subclassing :class:`typing.Generic`. From c68acb1384a51eb745f572687eaf677371b9e765 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 9 May 2024 12:17:02 +0300 Subject: [PATCH 018/903] gh-118798: Remove deprecated isdst parameter from `email.utils.localtime` (#118799) --- Doc/library/email.utils.rst | 3 +-- Doc/whatsnew/3.14.rst | 9 +++++++++ Lib/email/utils.py | 10 +--------- Lib/test/test_email/test_utils.py | 4 ---- .../2024-05-08-23-16-50.gh-issue-118798.Q_ybqP.rst | 2 ++ 5 files changed, 13 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-08-23-16-50.gh-issue-118798.Q_ybqP.rst diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 6f0bed130bc64c..43e5b25df01f79 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -17,8 +17,7 @@ module: arguments, return current time. Otherwise *dt* argument should be a :class:`~datetime.datetime` instance, and it is converted to the local time zone according to the system time zone database. If *dt* is naive (that - is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. The - *isdst* parameter is ignored. + is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. .. versionadded:: 3.3 diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 496a5d8ae7bc5e..24b6b617fe17dc 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -101,6 +101,15 @@ Deprecated Removed ======= +email +----- + +* The *isdst* parameter has been removed from :func:`email.utils.localtime`. + (Contributed by Hugo van Kemenade in :gh:`118798`.) + +Others +------ + * Using :data:`NotImplemented` in a boolean context will now raise a :exc:`TypeError`. It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed by Jelle Zijlstra in :gh:`118767`.) diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 103cef61a83538..6d897ca8eeee91 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -466,23 +466,15 @@ def collapse_rfc2231_value(value, errors='replace', # better than not having it. # -def localtime(dt=None, isdst=None): +def localtime(dt=None): """Return local time as an aware datetime object. If called without arguments, return current time. Otherwise *dt* argument should be a datetime instance, and it is converted to the local time zone according to the system time zone database. If *dt* is naive (that is, dt.tzinfo is None), it is assumed to be in local time. - The isdst parameter is ignored. """ - if isdst is not None: - import warnings - warnings._deprecated( - "The 'isdst' parameter to 'localtime'", - message='{name} is deprecated and slated for removal in Python {remove}', - remove=(3, 14), - ) if dt is None: dt = datetime.datetime.now() return dt.astimezone() diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py index d04b3909efa643..8556a93699d194 100644 --- a/Lib/test/test_email/test_utils.py +++ b/Lib/test/test_email/test_utils.py @@ -154,10 +154,6 @@ def test_variable_tzname(self): t1 = utils.localtime(t0) self.assertEqual(t1.tzname(), 'EET') - def test_isdst_deprecation(self): - with self.assertWarns(DeprecationWarning): - t0 = datetime.datetime(1990, 1, 1) - t1 = utils.localtime(t0, isdst=True) # Issue #24836: The timezone files are out of date (pre 2011k) # on Mac OS X Snow Leopard. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-23-16-50.gh-issue-118798.Q_ybqP.rst b/Misc/NEWS.d/next/Library/2024-05-08-23-16-50.gh-issue-118798.Q_ybqP.rst new file mode 100644 index 00000000000000..28847e13207ffe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-23-16-50.gh-issue-118798.Q_ybqP.rst @@ -0,0 +1,2 @@ +The *isdst* parameter has been removed from :func:`email.utils.localtime`. +Patch by Hugo van Kemenade. From da090f1658e72485b201507653f6d673f3e39c86 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 9 May 2024 14:46:45 +0300 Subject: [PATCH 019/903] gh-118805: Remove type, choices, metavar params of `BooleanOptionalAction` (#118806) Co-authored-by: Alex Waygood --- Doc/whatsnew/3.14.rst | 7 +++ Lib/argparse.py | 28 ------------ Lib/test/test_argparse.py | 43 ------------------- ...-05-09-01-05-52.gh-issue-118805.N7dm07.rst | 3 ++ 4 files changed, 10 insertions(+), 71 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-09-01-05-52.gh-issue-118805.N7dm07.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 24b6b617fe17dc..25c43dc0387eaf 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -101,6 +101,13 @@ Deprecated Removed ======= +argparse +-------- + +* The *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction` are removed. + They were deprecated since 3.12. + email ----- diff --git a/Lib/argparse.py b/Lib/argparse.py index 55bf8cdd875a8d..cdd29d3ad568e5 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -831,19 +831,13 @@ def __call__(self, parser, namespace, values, option_string=None): raise NotImplementedError(_('.__call__() not defined')) -# FIXME: remove together with `BooleanOptionalAction` deprecated arguments. -_deprecated_default = object() - class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, default=None, - type=_deprecated_default, - choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default, deprecated=False): _option_strings = [] @@ -854,35 +848,13 @@ def __init__(self, option_string = '--no-' + option_string[2:] _option_strings.append(option_string) - # We need `_deprecated` special value to ban explicit arguments that - # match default value. Like: - # parser.add_argument('-f', action=BooleanOptionalAction, type=int) - for field_name in ('type', 'choices', 'metavar'): - if locals()[field_name] is not _deprecated_default: - import warnings - warnings._deprecated( - field_name, - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - if type is _deprecated_default: - type = None - if choices is _deprecated_default: - choices = None - if metavar is _deprecated_default: - metavar = None - super().__init__( option_strings=_option_strings, dest=dest, nargs=0, default=default, - type=type, - choices=choices, required=required, help=help, - metavar=metavar, deprecated=deprecated) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 02b499145f6c43..eb1a9f5146beb4 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -765,49 +765,6 @@ def test_const(self): self.assertIn("got an unexpected keyword argument 'const'", str(cm.exception)) - def test_deprecated_init_kw(self): - # See gh-92248 - parser = argparse.ArgumentParser() - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-a', - action=argparse.BooleanOptionalAction, - type=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-b', - action=argparse.BooleanOptionalAction, - type=bool, - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-c', - action=argparse.BooleanOptionalAction, - metavar=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-d', - action=argparse.BooleanOptionalAction, - metavar='d', - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-e', - action=argparse.BooleanOptionalAction, - choices=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-f', - action=argparse.BooleanOptionalAction, - choices=(), - ) - class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" diff --git a/Misc/NEWS.d/next/Library/2024-05-09-01-05-52.gh-issue-118805.N7dm07.rst b/Misc/NEWS.d/next/Library/2024-05-09-01-05-52.gh-issue-118805.N7dm07.rst new file mode 100644 index 00000000000000..4f1db04d8bd67f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-01-05-52.gh-issue-118805.N7dm07.rst @@ -0,0 +1,3 @@ +Remove *type*, *choices*, and *metavar* parameters of +:class:`!argparse.BooleanOptionalAction`. +They were deprecated since Python 3.12. From 7c87ce777b3fd9055b118a58ec8614901ecb45e9 Mon Sep 17 00:00:00 2001 From: Radislav Chugunov <52372310+chgnrdv@users.noreply.github.com> Date: Thu, 9 May 2024 15:09:44 +0300 Subject: [PATCH 020/903] gh-103956: Fix `trace` output in case of missing source line (GH-103958) Print only filename with lineno if linecache.getline() returns an empty string. --- Lib/test/test_trace.py | 25 +++++++++++++++++++ Lib/trace.py | 16 +++++++++--- ...-04-28-09-54-15.gh-issue-103956.EyLDPS.rst | 1 + 3 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-28-09-54-15.gh-issue-103956.EyLDPS.rst diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index c1e289bcaff9e5..93966ee31d0a01 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -6,6 +6,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure import textwrap import unittest +from types import FunctionType import trace from trace import Trace @@ -559,5 +560,29 @@ def test_run_as_module(self): assert_python_failure('-m', 'trace', '-l', '--module', 'not_a_module_zzz') +class TestTrace(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + self.tracer = Trace(count=0, trace=1) + self.filemod = my_file_and_modname() + + def test_no_source_file(self): + filename = "" + co = traced_func_linear.__code__ + co = co.replace(co_filename=filename) + f = FunctionType(co, globals()) + + with captured_stdout() as out: + self.tracer.runfunc(f, 2, 3) + + out = out.getvalue().splitlines() + firstlineno = get_firstlineno(f) + self.assertIn(f" --- modulename: {self.filemod[1]}, funcname: {f.__code__.co_name}", out[0]) + self.assertIn(f"{filename}({firstlineno + 1})", out[1]) + self.assertIn(f"{filename}({firstlineno + 2})", out[2]) + self.assertIn(f"{filename}({firstlineno + 3})", out[3]) + self.assertIn(f"{filename}({firstlineno + 4})", out[4]) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/trace.py b/Lib/trace.py index 7886959fa64f68..64fc8037e355ee 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -565,8 +565,12 @@ def localtrace_trace_and_count(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_trace(self, frame, why, arg): @@ -578,8 +582,12 @@ def localtrace_trace(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_count(self, frame, why, arg): diff --git a/Misc/NEWS.d/next/Library/2023-04-28-09-54-15.gh-issue-103956.EyLDPS.rst b/Misc/NEWS.d/next/Library/2023-04-28-09-54-15.gh-issue-103956.EyLDPS.rst new file mode 100644 index 00000000000000..4ce1491ffa91e2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-28-09-54-15.gh-issue-103956.EyLDPS.rst @@ -0,0 +1 @@ +Fix lack of newline characters in :mod:`trace` module output when line tracing is enabled but source code line for current frame is not available. From 82acc5f2113bffd0ed902851f4ccf5b9be8980b2 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 9 May 2024 13:59:18 +0100 Subject: [PATCH 021/903] gh-118802: Fix ACL use in test for non-English Windows (GH-118831) --- Lib/test/test_os.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 9c9c8536dc7542..9088318600f4c0 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3226,9 +3226,8 @@ def test_stat_inaccessible_file(self): self.skipTest("Unable to create inaccessible file") def cleanup(): - # Give delete permission. We are the file owner, so we can do this - # even though we removed all permissions earlier. - subprocess.check_output([ICACLS, filename, "/grant", "Everyone:(D)"], + # Give delete permission to the owner (us) + subprocess.check_output([ICACLS, filename, "/grant", "*WD:(D)"], stderr=subprocess.STDOUT) os.unlink(filename) From 35b5eaa176a5bae8a0cacb9c9f40ec948ecc4325 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 9 May 2024 06:52:08 -0700 Subject: [PATCH 022/903] gh-118767: Improve tests and docs for bool(NotImplemented) (#118813) --- Doc/library/constants.rst | 6 +++--- Doc/reference/datamodel.rst | 6 +++--- Lib/test/test_builtin.py | 14 ++++++++------ .../2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 890517c3eb68dc..6c1063cda6690e 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -53,12 +53,12 @@ A small number of constants live in the built-in namespace. They are: See :exc:`NotImplementedError` for details on when to use it. .. versionchanged:: 3.9 - Evaluating :data:`!NotImplemented` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. + Evaluating :data:`!NotImplemented` in a boolean context was deprecated. .. versionchanged:: 3.14 Evaluating :data:`!NotImplemented` in a boolean context now raises a :exc:`TypeError`. + It previously evaluated to :const:`True` and emitted a :exc:`DeprecationWarning` + since Python 3.9. .. index:: single: ...; ellipsis literal diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index fc072b4e75314d..d3e066797f8837 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -170,12 +170,12 @@ See for more details. .. versionchanged:: 3.9 - Evaluating :data:`NotImplemented` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. + Evaluating :data:`NotImplemented` in a boolean context was deprecated. .. versionchanged:: 3.14 Evaluating :data:`NotImplemented` in a boolean context now raises a :exc:`TypeError`. + It previously evaluated to :const:`True` and emitted a :exc:`DeprecationWarning` + since Python 3.9. Ellipsis diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 120814379dd53a..a7631f92e7ea81 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2125,15 +2125,17 @@ def test_construct_singletons(self): self.assertRaises(TypeError, tp, 1, 2) self.assertRaises(TypeError, tp, a=1, b=2) - def test_warning_notimplemented(self): - # Issue #35712: NotImplemented is a sentinel value that should never + def test_bool_notimplemented(self): + # GH-79893: NotImplemented is a sentinel value that should never # be evaluated in a boolean context (virtually all such use cases # are a result of accidental misuse implementing rich comparison # operations in terms of one another). - self.assertRaises(TypeError, bool, NotImplemented) - with self.assertRaises(TypeError): - self.assertTrue(NotImplemented) - with self.assertRaises(TypeError): + msg = "NotImplemented should not be used in a boolean context" + self.assertRaisesRegex(TypeError, msg, bool, NotImplemented) + with self.assertRaisesRegex(TypeError, msg): + if NotImplemented: + pass + with self.assertRaisesRegex(TypeError, msg): not NotImplemented diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst index 76548effd1449f..4828f8fbf50cea 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-08-09-44-15.gh-issue-118767.iFF5F5.rst @@ -1,2 +1,2 @@ Using :data:`NotImplemented` in a boolean context now raises -:exc:`TypeError`. Contributed by Jelle Zijlstra in :gh:`118767`. +:exc:`TypeError`. Contributed by Jelle Zijlstra. From c3643a121401d111bebd3e26d6f362ade2ed2a83 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 9 May 2024 18:20:46 +0300 Subject: [PATCH 023/903] gh-118817: Fix `asyncio REPL` on Windows (#118819) --- Lib/asyncio/__main__.py | 7 ++++--- Lib/test/test_repl.py | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index cbc1d7c93ef76f..9041b8b8316c1e 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -108,7 +108,7 @@ def run(self): try: import readline # NoQA except ImportError: - pass + readline = None interactive_hook = getattr(sys, "__interactivehook__", None) @@ -122,8 +122,9 @@ def run(self): except: pass else: - completer = rlcompleter.Completer(console.locals) - readline.set_completer(completer.complete) + if readline is not None: + completer = rlcompleter.Completer(console.locals) + readline.set_completer(completer.complete) repl_thread = REPLThread() repl_thread.daemon = True diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 457279a4db687d..340178366fc13a 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -7,7 +7,7 @@ from textwrap import dedent from test import support from test.support import cpython_only, has_subprocess_support, SuppressCrashReport -from test.support.script_helper import kill_python +from test.support.script_helper import kill_python, assert_python_ok from test.support.import_helper import import_module @@ -195,6 +195,9 @@ def bar(x): expected = "(30, None, [\'def foo(x):\\n\', \' return x + 1\\n\', \'\\n\'], \'\')" self.assertIn(expected, output, expected) + def test_asyncio_repl_is_ok(self): + assert_python_ok("-m", "asyncio") + class TestInteractiveModeSyntaxErrors(unittest.TestCase): From c30d8e5d6c6b657817d6b342f1021676d04dd5af Mon Sep 17 00:00:00 2001 From: mpage Date: Thu, 9 May 2024 09:05:52 -0700 Subject: [PATCH 024/903] gh-117657: Acquire a critical section around `SemLock.__{enter,exit}__` (#118812) These methods are purely wrappers around `Semlock.{acquire,release}`, which expect a critical section to be held. --- Modules/_multiprocessing/clinic/semaphore.c.h | 12 ++++++++++-- Modules/_multiprocessing/semaphore.c | 6 ++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index 64e666b5af6f5b..512e5a016192fb 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -473,7 +473,13 @@ _multiprocessing_SemLock___enter___impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock___enter__(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock___enter___impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock___enter___impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -518,7 +524,9 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py } exc_tb = args[2]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multiprocessing_SemLock___exit___impl(self, exc_type, exc_value, exc_tb); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -565,4 +573,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=713b597256233716 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dea36482d23a355f input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index 5bb055f501e35b..4de4ee6c78fbd1 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -682,6 +682,7 @@ _multiprocessing_SemLock__after_fork_impl(SemLockObject *self) } /*[clinic input] +@critical_section _multiprocessing.SemLock.__enter__ Enter the semaphore/lock. @@ -689,12 +690,13 @@ Enter the semaphore/lock. static PyObject * _multiprocessing_SemLock___enter___impl(SemLockObject *self) -/*[clinic end generated code: output=beeb2f07c858511f input=c5e27d594284690b]*/ +/*[clinic end generated code: output=beeb2f07c858511f input=d35c9860992ee790]*/ { return _multiprocessing_SemLock_acquire_impl(self, 1, Py_None); } /*[clinic input] +@critical_section _multiprocessing.SemLock.__exit__ exc_type: object = None @@ -709,7 +711,7 @@ static PyObject * _multiprocessing_SemLock___exit___impl(SemLockObject *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb) -/*[clinic end generated code: output=3b37c1a9f8b91a03 input=7d644b64a89903f8]*/ +/*[clinic end generated code: output=3b37c1a9f8b91a03 input=1610c8cc3e0e337e]*/ { return _multiprocessing_SemLock_release_impl(self); } From 8af84b503d0b62a3db0d806d39f42c1e08746079 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 9 May 2024 17:43:21 +0100 Subject: [PATCH 025/903] gh-118773: Use language-invariant SDDL string instead of aliases for ACLs. (GH-118800) --- ...-05-08-21-59-38.gh-issue-118773.7dFRJY.rst | 2 + Modules/posixmodule.c | 176 +++--------------- 2 files changed, 24 insertions(+), 154 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst diff --git a/Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst b/Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst new file mode 100644 index 00000000000000..bfec178f6318a7 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst @@ -0,0 +1,2 @@ +Fixes creation of ACLs in :func:`os.mkdir` on Windows to work correctly on +non-English machines. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9f4be98b35186e..5fe6036b3817e4 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5541,146 +5541,6 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return result; } -#ifdef MS_WINDOWS - -/* We centralise SECURITY_ATTRIBUTE initialization based around -templates that will probably mostly match common POSIX mode settings. -The _Py_SECURITY_ATTRIBUTE_DATA structure contains temporary data, as -a constructed SECURITY_ATTRIBUTE structure typically refers to memory -that has to be alive while it's being used. - -Typical use will look like: - SECURITY_ATTRIBUTES *pSecAttr = NULL; - struct _Py_SECURITY_ATTRIBUTE_DATA secAttrData; - int error, error2; - - Py_BEGIN_ALLOW_THREADS - switch (mode) { - case 0x1C0: // 0o700 - error = initializeMkdir700SecurityAttributes(&pSecAttr, &secAttrData); - break; - ... - default: - error = initializeDefaultSecurityAttributes(&pSecAttr, &secAttrData); - break; - } - - if (!error) { - // do operation, passing pSecAttr - } - - // Unconditionally clear secAttrData. - error2 = clearSecurityAttributes(&pSecAttr, &secAttrData); - if (!error) { - error = error2; - } - Py_END_ALLOW_THREADS - - if (error) { - PyErr_SetFromWindowsErr(error); - return NULL; - } -*/ - -struct _Py_SECURITY_ATTRIBUTE_DATA { - SECURITY_ATTRIBUTES securityAttributes; - PACL acl; - SECURITY_DESCRIPTOR sd; - EXPLICIT_ACCESS_W ea[4]; - char sid[64]; -}; - -static int -initializeDefaultSecurityAttributes( - PSECURITY_ATTRIBUTES *securityAttributes, - struct _Py_SECURITY_ATTRIBUTE_DATA *data -) { - assert(securityAttributes); - assert(data); - *securityAttributes = NULL; - memset(data, 0, sizeof(*data)); - return 0; -} - -static int -initializeMkdir700SecurityAttributes( - PSECURITY_ATTRIBUTES *securityAttributes, - struct _Py_SECURITY_ATTRIBUTE_DATA *data -) { - assert(securityAttributes); - assert(data); - *securityAttributes = NULL; - memset(data, 0, sizeof(*data)); - - if (!InitializeSecurityDescriptor(&data->sd, SECURITY_DESCRIPTOR_REVISION) - || !SetSecurityDescriptorGroup(&data->sd, NULL, TRUE)) { - return GetLastError(); - } - - int use_alias = 0; - DWORD cbSid = sizeof(data->sid); - if (!CreateWellKnownSid(WinCreatorOwnerRightsSid, NULL, (PSID)data->sid, &cbSid)) { - use_alias = 1; - } - - data->securityAttributes.nLength = sizeof(SECURITY_ATTRIBUTES); - data->ea[0].grfAccessPermissions = GENERIC_ALL; - data->ea[0].grfAccessMode = SET_ACCESS; - data->ea[0].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; - if (use_alias) { - data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_NAME; - data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; - data->ea[0].Trustee.ptstrName = L"CURRENT_USER"; - } else { - data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_SID; - data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP; - data->ea[0].Trustee.ptstrName = (LPWCH)(SID*)data->sid; - } - - data->ea[1].grfAccessPermissions = GENERIC_ALL; - data->ea[1].grfAccessMode = SET_ACCESS; - data->ea[1].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; - data->ea[1].Trustee.TrusteeForm = TRUSTEE_IS_NAME; - data->ea[1].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; - data->ea[1].Trustee.ptstrName = L"SYSTEM"; - - data->ea[2].grfAccessPermissions = GENERIC_ALL; - data->ea[2].grfAccessMode = SET_ACCESS; - data->ea[2].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; - data->ea[2].Trustee.TrusteeForm = TRUSTEE_IS_NAME; - data->ea[2].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; - data->ea[2].Trustee.ptstrName = L"ADMINISTRATORS"; - - int r = SetEntriesInAclW(3, data->ea, NULL, &data->acl); - if (r) { - return r; - } - if (!SetSecurityDescriptorDacl(&data->sd, TRUE, data->acl, FALSE)) { - return GetLastError(); - } - data->securityAttributes.lpSecurityDescriptor = &data->sd; - *securityAttributes = &data->securityAttributes; - return 0; -} - -static int -clearSecurityAttributes( - PSECURITY_ATTRIBUTES *securityAttributes, - struct _Py_SECURITY_ATTRIBUTE_DATA *data -) { - assert(securityAttributes); - assert(data); - *securityAttributes = NULL; - if (data->acl) { - if (LocalFree((void *)data->acl)) { - return GetLastError(); - } - } - return 0; -} - -#endif - /*[clinic input] os.mkdir @@ -5713,8 +5573,8 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #ifdef MS_WINDOWS int error = 0; int pathError = 0; + SECURITY_ATTRIBUTES secAttr = { sizeof(secAttr) }; SECURITY_ATTRIBUTES *pSecAttr = NULL; - struct _Py_SECURITY_ATTRIBUTE_DATA secAttrData; #endif #ifdef HAVE_MKDIRAT int mkdirat_unavailable = 0; @@ -5727,26 +5587,34 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS - switch (mode) { - case 0x1C0: // 0o700 - error = initializeMkdir700SecurityAttributes(&pSecAttr, &secAttrData); - break; - default: - error = initializeDefaultSecurityAttributes(&pSecAttr, &secAttrData); - break; + if (mode == 0700 /* 0o700 */) { + ULONG sdSize; + pSecAttr = &secAttr; + // Set a discreationary ACL (D) that is protected (P) and includes + // inheritable (OICI) entries that allow (A) full control (FA) to + // SYSTEM (SY), Administrators (BA), and the owner (OW). + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + L"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)", + SDDL_REVISION_1, + &secAttr.lpSecurityDescriptor, + &sdSize + )) { + error = GetLastError(); + } } if (!error) { result = CreateDirectoryW(path->wide, pSecAttr); - error = clearSecurityAttributes(&pSecAttr, &secAttrData); - } else { - // Ignore error from "clear" - we have a more interesting one already - clearSecurityAttributes(&pSecAttr, &secAttrData); + if (secAttr.lpSecurityDescriptor && + // uncommonly, LocalFree returns non-zero on error, but still uses + // GetLastError() to see what the error code is + LocalFree(secAttr.lpSecurityDescriptor)) { + error = GetLastError(); + } } Py_END_ALLOW_THREADS if (error) { - PyErr_SetFromWindowsErr(error); - return NULL; + return PyErr_SetFromWindowsErr(error); } if (!result) { return path_error(path); From 82abe75e77129bebb3c13d807e8040f6924194f6 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 9 May 2024 20:15:14 +0300 Subject: [PATCH 026/903] gh-118849: Fix "code will never be executed" warning in `dictobject.c` (#118850) --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b0fce09d7940e0..985a326a176c94 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5396,6 +5396,7 @@ static int dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, PyObject **out_key, PyObject **out_value) { + int res; dictiterobject *di = (dictiterobject *)self; Py_ssize_t i; PyDictKeysObject *k; @@ -5491,7 +5492,6 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, Py_DECREF(d); return -1; - int res; try_locked: Py_BEGIN_CRITICAL_SECTION(d); res = dictiter_iternextitem_lock_held(d, self, out_key, out_value); From 2402715e10d00ef60fad2948d8461559d084eb36 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 14:52:27 -0400 Subject: [PATCH 027/903] gh-118561: Fix crash involving list.extend in free-threaded build (#118723) The `list_preallocate_exact` function did not zero initialize array contents. In the free-threaded build, this could expose uninitialized memory to concurrent readers between the call to `list_preallocate_exact` and the filling of the array contents with items. --- .../2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst | 2 ++ Objects/listobject.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst new file mode 100644 index 00000000000000..9eaf0abb8a6128 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst @@ -0,0 +1,2 @@ +Fix race condition in free-threaded build where :meth:`list.extend` could expose +uninitialied memory to concurrent readers. diff --git a/Objects/listobject.c b/Objects/listobject.c index 3c4e2d2e6ed7de..7070165014f137 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -192,6 +192,7 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) return -1; } items = array->ob_item; + memset(items, 0, size * sizeof(PyObject *)); #else items = PyMem_New(PyObject*, size); if (items == NULL) { @@ -199,7 +200,7 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) return -1; } #endif - self->ob_item = items; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, items); self->allocated = size; return 0; } From 71cc0651e79041abd648595f3030dfa41009137a Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 16:03:45 -0400 Subject: [PATCH 028/903] gh-116984: Make mimalloc header includes relative to the current file (#118808) Some embedders and extensions include parts of the internal API. The pycore_mimalloc.h file is transitively include by a number of other internal headers. This avoids include errors for code that was already including those headers. --- Include/internal/mimalloc/mimalloc/internal.h | 4 ++-- Include/internal/mimalloc/mimalloc/types.h | 2 +- Include/internal/pycore_mimalloc.h | 6 +++--- .../C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst | 3 +++ 4 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index 8af841cfdffc01..94f88fb603af25 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -14,8 +14,8 @@ terms of the MIT license. A copy of the license can be found in the file // functions and macros. // -------------------------------------------------------------------------- -#include "mimalloc/types.h" -#include "mimalloc/track.h" +#include "types.h" +#include "track.h" #if (MI_DEBUG>0) #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 17e440848ecae5..354839ba955b36 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -21,7 +21,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // ptrdiff_t #include // uintptr_t, uint16_t, etc -#include "mimalloc/atomic.h" // _Atomic +#include "atomic.h" // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index 10d451398f1410..100f78d53021ee 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -36,9 +36,9 @@ typedef enum { # define MI_TSAN 1 #endif -#include "mimalloc.h" -#include "mimalloc/types.h" -#include "mimalloc/internal.h" +#include "mimalloc/mimalloc.h" +#include "mimalloc/mimalloc/types.h" +#include "mimalloc/mimalloc/internal.h" #endif #ifdef Py_GIL_DISABLED diff --git a/Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst b/Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst new file mode 100644 index 00000000000000..561417b80d444d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst @@ -0,0 +1,3 @@ +Make mimalloc includes relative to the current file to avoid embedders or +extensions needing to include ``Internal/mimalloc`` if they are already +including internal CPython headers. From 1b1db2fd9a531e26b79b34667bccfb938c4d184d Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 16:06:20 -0400 Subject: [PATCH 029/903] gh-118846: Fix PGO tests in free-threaded build (#118862) Avoid immortalizing objects in tests that verify garbage collection of classes or modules. This fixes test_ordered_dict and test_struct. --- Lib/test/test_ordered_dict.py | 3 ++- Lib/test/test_struct.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 4571b23dfe7c1a..06a0e81227188c 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -10,7 +10,7 @@ import weakref from collections.abc import MutableMapping from test import mapping_tests, support -from test.support import import_helper +from test.support import import_helper, suppress_immortalization py_coll = import_helper.import_fresh_module('collections', @@ -667,6 +667,7 @@ def test_dict_update(self): dict.update(od, [('spam', 1)]) self.assertNotIn('NULL', repr(od)) + @suppress_immortalization() def test_reference_loop(self): # Issue 25935 OrderedDict = self.OrderedDict diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 15f6ee06ffe19b..5508cc3eec85c8 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -9,7 +9,7 @@ import weakref from test import support -from test.support import import_helper +from test.support import import_helper, suppress_immortalization from test.support.script_helper import assert_python_ok ISBIGENDIAN = sys.byteorder == "big" @@ -674,6 +674,7 @@ def __del__(self): self.assertIn(b"Exception ignored in:", stderr) self.assertIn(b"C.__del__", stderr) + @suppress_immortalization() def test__struct_reference_cycle_cleaned_up(self): # Regression test for python/cpython#94207. From 98ff3f65c0232f31df89ebb52b244625ec9e3eb6 Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Thu, 9 May 2024 14:02:39 -0700 Subject: [PATCH 030/903] gh-117657: Replace TSAN suppresions with more specific rules (#118722) Using `race:` filters out warnings if the function appears anywhere in the stack trace. This can hide a lot of unrelated warnings, especially for a function like `_PyEval_EvalFrameDefault`, which is somewhere on the stack more often than not. Change all free-threaded suppressions to `race_top:`, which only matches the top frame, and add any new suppressions this exposes. --- Tools/tsan/suppressions_free_threading.txt | 105 ++++++++++++++++----- 1 file changed, 81 insertions(+), 24 deletions(-) diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index a669bc4d1d5c30..7f91a9113c4e1a 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -11,31 +11,88 @@ race:set_allocator_unlocked ## Free-threaded suppressions -race:_add_to_weak_set -race:_in_weak_set -race:_mi_heap_delayed_free_partial -race:_PyEval_EvalFrameDefault -race:_PyImport_AcquireLock -race:_PyImport_ReleaseLock -race:_PyInterpreterState_SetNotRunningMain -race:_PyInterpreterState_IsRunningMain -# https://gist.github.com/mpage/0a24eb2dd458441ededb498e9b0e5de8 + +# These entries are for warnings that trigger in a library function, as called +# by a CPython function. + +# https://gist.github.com/swtaarrs/9d41251e603fa6dedd604191a6da820d +race:park_detached_threads +# https://gist.github.com/swtaarrs/8e0e365e1d9cecece3269a2fb2f2b8b8 +race:sock_recv_impl +# https://gist.github.com/swtaarrs/08dfe7883b4c975c31ecb39388987a67 +race:free_threadstate +# https://gist.github.com/swtaarrs/cd6aec2006e0c1b561b68d65e9f1a872 race:_PyParkingLot_Park -race:_PyType_HasFeature -race:assign_version_tag -race:gc_restore_tid -race:initialize_new_array -race:insertdict -race:lookup_tp_dict -race:mi_heap_visit_pages -race:PyMember_GetOne -race:PyMember_SetOne -race:new_reference -race:set_contains_key -race:set_inheritable -race:start_the_world -race:tstate_set_detached -race:unicode_hash + + +# These warnings trigger directly in a CPython function. + +race_top:_add_to_weak_set +race_top:_in_weak_set +race_top:_mi_heap_delayed_free_partial +race_top:_PyEval_EvalFrameDefault +race_top:_PyImport_AcquireLock +race_top:_PyImport_ReleaseLock +race_top:_PyInterpreterState_SetNotRunningMain +race_top:_PyInterpreterState_IsRunningMain +# https://gist.github.com/mpage/0a24eb2dd458441ededb498e9b0e5de8 +race_top:_PyParkingLot_Park +race_top:_PyType_HasFeature +race_top:assign_version_tag +race_top:gc_restore_tid +race_top:initialize_new_array +race_top:insertdict +race_top:lookup_tp_dict +race_top:mi_heap_visit_pages +race_top:PyMember_GetOne +race_top:PyMember_SetOne +race_top:new_reference +race_top:set_contains_key +race_top:set_inheritable +race_top:start_the_world +race_top:tstate_set_detached +race_top:unicode_hash +race_top:Py_SET_TYPE +race_top:_PyDict_CheckConsistency +race_top:_PyImport_AcquireLock +race_top:_Py_dict_lookup_threadsafe +race_top:_imp_release_lock +race_top:_multiprocessing_SemLock_acquire_impl +race_top:builtin_compile_impl +race_top:count_next +race_top:dictiter_new +race_top:dictresize +race_top:insert_to_emptydict +race_top:insertdict +race_top:list_get_item_ref +race_top:make_pending_calls +race_top:set_add_entry +race_top:should_intern_string +race_top:worklist_pop +race_top:_PyEval_IsGILEnabled +race_top:llist_insert_tail +race_top:_Py_slot_tp_getattr_hook +race_top:add_threadstate +race_top:dump_traceback +race_top:fatal_error +race_top:mi_page_decode_padding +race_top:_multiprocessing_SemLock_release_impl +race_top:_PyFrame_GetCode +race_top:_PyFrame_Initialize +race_top:PyInterpreterState_ThreadHead +race_top:_PyObject_TryGetInstanceAttribute +race_top:_Py_qsbr_unregister +race_top:_Py_qsbr_poll +race_top:PyThreadState_Next +race_top:Py_TYPE +race_top:PyUnstable_InterpreterFrame_GetLine +race_top:sock_close +race_top:tstate_delete_common +race_top:tstate_is_freed +race_top:type_modified_unlocked +race_top:update_refs +race_top:write_thread_id +race_top:PyThreadState_Clear # https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40 thread:pthread_create From 68fbc00dc870f6a8dcbecd2ec19298e21015867f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 9 May 2024 15:30:14 -0700 Subject: [PATCH 031/903] gh-118851: Default ctx arguments to AST constructors to Load() (#118854) Co-authored-by: Alex Waygood --- Doc/library/ast.rst | 6 +++--- Doc/whatsnew/3.13.rst | 8 +++++--- Lib/test/test_ast.py | 17 +++++++++++++++++ ...24-05-09-08-46-12.gh-issue-118851.aPAoJw.rst | 2 ++ Parser/asdl_c.py | 7 +++++++ Python/Python-ast.c | 7 +++++++ 6 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 24c56f17ebb002..d4ccf282a5d00a 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -120,7 +120,8 @@ Node classes If a field that is optional in the grammar is omitted from the constructor, it defaults to ``None``. If a list field is omitted, it defaults to the empty - list. If any other field is omitted, a :exc:`DeprecationWarning` is raised + list. If a field of type :class:`!ast.expr_context` is omitted, it defaults to + :class:`Load() `. If any other field is omitted, a :exc:`DeprecationWarning` is raised and the AST node will not have this field. In Python 3.15, this condition will raise an error. @@ -596,8 +597,7 @@ Expressions * ``keywords`` holds a list of :class:`.keyword` objects representing arguments passed by keyword. - When creating a ``Call`` node, ``args`` and ``keywords`` are required, but - they can be empty lists. + The ``args`` and ``keywords`` arguments are optional and default to empty lists. .. doctest:: diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 44555718184e19..9dab458b210093 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -521,8 +521,10 @@ ast If an optional field on an AST node is not included as an argument when constructing an instance, the field will now be set to ``None``. Similarly, - if a list field is omitted, that field will now be set to an empty list. - (Previously, in both cases, the attribute would be missing on the newly + if a list field is omitted, that field will now be set to an empty list, + and if a :class:`!ast.expr_context` field is omitted, it defaults to + :class:`Load() `. + (Previously, in all cases, the attribute would be missing on the newly constructed AST node instance.) If other arguments are omitted, a :exc:`DeprecationWarning` is emitted. @@ -534,7 +536,7 @@ ast unless the class opts in to the new behavior by setting the attribute :attr:`ast.AST._field_types`. - (Contributed by Jelle Zijlstra in :gh:`105858` and :gh:`117486`.) + (Contributed by Jelle Zijlstra in :gh:`105858`, :gh:`117486`, and :gh:`118851`.) * :func:`ast.parse` now accepts an optional argument *optimize* which is passed on to the :func:`compile` built-in. This makes it diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index f6e22d44406d9e..5422c861ffb5c0 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3036,6 +3036,23 @@ def test_FunctionDef(self): self.assertEqual(node.name, 'foo') self.assertEqual(node.decorator_list, []) + def test_expr_context(self): + name = ast.Name("x") + self.assertEqual(name.id, "x") + self.assertIsInstance(name.ctx, ast.Load) + + name2 = ast.Name("x", ast.Store()) + self.assertEqual(name2.id, "x") + self.assertIsInstance(name2.ctx, ast.Store) + + name3 = ast.Name("x", ctx=ast.Del()) + self.assertEqual(name3.id, "x") + self.assertIsInstance(name3.ctx, ast.Del) + + with self.assertWarnsRegex(DeprecationWarning, + r"Name\.__init__ missing 1 required positional argument: 'id'"): + name3 = ast.Name() + def test_custom_subclass_with_no_fields(self): class NoInit(ast.AST): pass diff --git a/Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst b/Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst new file mode 100644 index 00000000000000..d036d0cda617ef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst @@ -0,0 +1,2 @@ +``ctx`` arguments to the constructors of :mod:`ast` node classes now default +to :class:`ast.Load() `. Patch by Jelle Zijlstra. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 11d59faeb0d42c..9961d23629abc5 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1022,6 +1022,13 @@ def visitModule(self, mod): goto set_remaining_cleanup; } } + else if (type == state->expr_context_type) { + // special case for expr_context: default to Load() + res = PyObject_SetAttr(self, name, state->Load_singleton); + if (res < 0) { + goto set_remaining_cleanup; + } + } else { // simple field (e.g., identifier) if (PyErr_WarnFormat( diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 4956d04f719de9..7aa1c5119d8f28 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5221,6 +5221,13 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto set_remaining_cleanup; } } + else if (type == state->expr_context_type) { + // special case for expr_context: default to Load() + res = PyObject_SetAttr(self, name, state->Load_singleton); + if (res < 0) { + goto set_remaining_cleanup; + } + } else { // simple field (e.g., identifier) if (PyErr_WarnFormat( From 46c808172fd3148e3397234b23674bf70734fb55 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 18:33:53 -0400 Subject: [PATCH 032/903] Revert "gh-115432: Add critical section variant that handles a NULL object (#115433)" (#118861) This reverts commit ad4f909e0e7890e027c4ae7fea74586667242ad3. The API ended up not being used. --- Include/internal/pycore_critical_section.h | 29 ------------------- .../test_critical_sections.c | 9 ------ 2 files changed, 38 deletions(-) diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 23b85c2f9e9bb2..573d09a09683ef 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -99,15 +99,6 @@ extern "C" { _PyCriticalSection_End(&_cs); \ } -# define Py_XBEGIN_CRITICAL_SECTION(op) \ - { \ - _PyCriticalSection _cs_opt = {0}; \ - _PyCriticalSection_XBegin(&_cs_opt, _PyObject_CAST(op)) - -# define Py_XEND_CRITICAL_SECTION() \ - _PyCriticalSection_XEnd(&_cs_opt); \ - } - # define Py_BEGIN_CRITICAL_SECTION2(a, b) \ { \ _PyCriticalSection2 _cs2; \ @@ -144,8 +135,6 @@ extern "C" { # define Py_BEGIN_CRITICAL_SECTION_MUT(mut) # define Py_BEGIN_CRITICAL_SECTION(op) # define Py_END_CRITICAL_SECTION() -# define Py_XBEGIN_CRITICAL_SECTION(op) -# define Py_XEND_CRITICAL_SECTION() # define Py_BEGIN_CRITICAL_SECTION2(a, b) # define Py_END_CRITICAL_SECTION2() # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) @@ -202,16 +191,6 @@ _PyCriticalSection_Begin(_PyCriticalSection *c, PyMutex *m) } } -static inline void -_PyCriticalSection_XBegin(_PyCriticalSection *c, PyObject *op) -{ -#ifdef Py_GIL_DISABLED - if (op != NULL) { - _PyCriticalSection_Begin(c, &_PyObject_CAST(op)->ob_mutex); - } -#endif -} - // Removes the top-most critical section from the thread's stack of critical // sections. If the new top-most critical section is inactive, then it is // resumed. @@ -234,14 +213,6 @@ _PyCriticalSection_End(_PyCriticalSection *c) _PyCriticalSection_Pop(c); } -static inline void -_PyCriticalSection_XEnd(_PyCriticalSection *c) -{ - if (c->mutex) { - _PyCriticalSection_End(c); - } -} - static inline void _PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) { diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index cdf8a70fc79ff3..1c0e049efafcb7 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -49,15 +49,6 @@ test_critical_sections(PyObject *self, PyObject *Py_UNUSED(args)) Py_END_CRITICAL_SECTION2(); assert_nogil(!PyMutex_IsLocked(&d2->ob_mutex)); - // Optional variant behaves the same if the object is non-NULL - Py_XBEGIN_CRITICAL_SECTION(d1); - assert_nogil(PyMutex_IsLocked(&d1->ob_mutex)); - Py_XEND_CRITICAL_SECTION(); - - // No-op - Py_XBEGIN_CRITICAL_SECTION(NULL); - Py_XEND_CRITICAL_SECTION(); - Py_DECREF(d2); Py_DECREF(d1); Py_RETURN_NONE; From 17a2cc199d5d8dd9db49ea596cf243e2217263c6 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Fri, 10 May 2024 16:11:50 +0800 Subject: [PATCH 033/903] Docs: fix typos in documentation (GH-118815) --- Doc/library/importlib.rst | 4 ++-- Doc/library/logging.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index b58ef359378e4f..2ec15dd171c18a 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1252,7 +1252,7 @@ find and load modules. be only a single binary per framework, and there can be no executable binary material outside the Frameworks folder. - To accomodate this requirement, when running on iOS, extension module + To accommodate this requirement, when running on iOS, extension module binaries are *not* packaged as ``.so`` files on ``sys.path``, but as individual standalone frameworks. To discover those frameworks, this loader is be registered against the ``.fwork`` file extension, with a ``.fwork`` @@ -1279,7 +1279,7 @@ find and load modules. When a module is loaded with this loader, the ``__file__`` for the module will report as the location of the ``.fwork`` file. This allows code to use - the ``__file__`` of a module as an anchor for file system traveral. + the ``__file__`` of a module as an anchor for file system traversal. However, the spec origin will reference the location of the *actual* binary in the ``.framework`` folder. diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index fb6ca38ba72aba..564b34bcf1bb37 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1204,7 +1204,7 @@ functions. most programs will want to carefully and explicitly control the logging configuration, and should therefore prefer creating a module-level logger and calling :meth:`Logger.debug` (or other level-specific methods) on it, as - described at the beginnning of this documentation. + described at the beginning of this documentation. .. function:: info(msg, *args, **kwargs) From 7e6fcab20003b07621dc02ea78d6ea2fda500371 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 May 2024 10:31:55 +0100 Subject: [PATCH 034/903] Fix some missing null checks. (GH-118721) --- Objects/typeobject.c | 13 ++++++++----- PC/launcher2.c | 5 +++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4b144fab5de8f1..b7c3fcf47f23fc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6036,15 +6036,19 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } comma_w_quotes_sep = PyUnicode_FromString("', '"); + if (!comma_w_quotes_sep) { + Py_DECREF(sorted_methods); + return NULL; + } joined = PyUnicode_Join(comma_w_quotes_sep, sorted_methods); - method_count = PyObject_Length(sorted_methods); - Py_DECREF(sorted_methods); + Py_DECREF(comma_w_quotes_sep); if (joined == NULL) { - Py_DECREF(comma_w_quotes_sep); + Py_DECREF(sorted_methods); return NULL; } + method_count = PyObject_Length(sorted_methods); + Py_DECREF(sorted_methods); if (method_count == -1) { - Py_DECREF(comma_w_quotes_sep); Py_DECREF(joined); return NULL; } @@ -6056,7 +6060,6 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) method_count > 1 ? "s" : "", joined); Py_DECREF(joined); - Py_DECREF(comma_w_quotes_sep); return NULL; } PyObject *obj = type->tp_alloc(type, 0); diff --git a/PC/launcher2.c b/PC/launcher2.c index 139aa61bbe5cc2..98231613efb26f 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -2707,6 +2707,11 @@ process(int argc, wchar_t ** argv) DWORD len = GetEnvironmentVariableW(L"PYLAUNCHER_LIMIT_TO_COMPANY", NULL, 0); if (len > 1) { wchar_t *limitToCompany = allocSearchInfoBuffer(&search, len); + if (!limitToCompany) { + exitCode = RC_NO_MEMORY; + winerror(0, L"Failed to allocate internal buffer"); + goto abort; + } search.limitToCompany = limitToCompany; if (0 == GetEnvironmentVariableW(L"PYLAUNCHER_LIMIT_TO_COMPANY", limitToCompany, len)) { exitCode = RC_INTERNAL_ERROR; From e85e8deaf3220c8d12b69294e45645aaf20187b9 Mon Sep 17 00:00:00 2001 From: Dobatymo Date: Fri, 10 May 2024 17:47:30 +0800 Subject: [PATCH 035/903] gh-118209: Add Windows structured exception handling to mmap module (GH-118213) --- Doc/whatsnew/3.13.rst | 3 + Lib/test/test_mmap.py | 77 ++++ ...-04-24-05-16-32.gh-issue-118209.Ryyzlz.rst | 2 + Modules/mmapmodule.c | 376 +++++++++++++++--- 4 files changed, 410 insertions(+), 48 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-04-24-05-16-32.gh-issue-118209.Ryyzlz.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 9dab458b210093..37c857dd8197e5 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -796,6 +796,9 @@ mmap * :class:`mmap.mmap` now has a *trackfd* parameter on Unix; if it is ``False``, the file descriptor specified by *fileno* will not be duplicated. (Contributed by Zackery Spytz and Petr Viktorin in :gh:`78502`.) +* :class:`mmap.mmap` is now protected from crashing on Windows when the mapped memory + is inaccessible due to file system errors or access violations. + (Contributed by Jannis Weigend in :gh:`118209`.) opcode ------ diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index ee86227e026b67..a1cf5384ada5b5 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -3,6 +3,7 @@ ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink +from test.support.script_helper import assert_python_ok import unittest import errno import os @@ -12,6 +13,7 @@ import socket import string import sys +import textwrap import weakref # Skip test if we can't import mmap. @@ -1058,6 +1060,81 @@ def __exit__(self, exc_type, exc_value, traceback): with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): m.write_byte(X()) + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + @unittest.skipUnless(hasattr(mmap.mmap, '_protect'), 'test needs debug build') + def test_access_violations(self): + from test.support.os_helper import TESTFN + + code = textwrap.dedent(""" + import faulthandler + import mmap + import os + import sys + from contextlib import suppress + + # Prevent logging access violations to stderr. + faulthandler.disable() + + PAGESIZE = mmap.PAGESIZE + PAGE_NOACCESS = 0x01 + + with open(sys.argv[1], 'bw+') as f: + f.write(b'A'* PAGESIZE) + f.flush() + + m = mmap.mmap(f.fileno(), PAGESIZE) + m._protect(PAGE_NOACCESS, 0, PAGESIZE) + with suppress(OSError): + m.read(PAGESIZE) + assert False, 'mmap.read() did not raise' + with suppress(OSError): + m.read_byte() + assert False, 'mmap.read_byte() did not raise' + with suppress(OSError): + m.readline() + assert False, 'mmap.readline() did not raise' + with suppress(OSError): + m.write(b'A'* PAGESIZE) + assert False, 'mmap.write() did not raise' + with suppress(OSError): + m.write_byte(0) + assert False, 'mmap.write_byte() did not raise' + with suppress(OSError): + m[0] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0:10] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0:10:2] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0] = 1 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m[0:10] = b'A'* 10 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m[0:10:2] = b'A'* 5 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m.move(0, 10, 1) + assert False, 'mmap.move() did not raise' + with suppress(OSError): + list(m) # test mmap_item + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m.find(b'A') + assert False, 'mmap.find() did not raise' + with suppress(OSError): + m.rfind(b'A') + assert False, 'mmap.rfind() did not raise' + """) + rt, stdout, stderr = assert_python_ok("-c", code, TESTFN) + self.assertEqual(stdout.strip(), b'') + self.assertEqual(stderr.strip(), b'') + + class LargeMmapTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Windows/2024-04-24-05-16-32.gh-issue-118209.Ryyzlz.rst b/Misc/NEWS.d/next/Windows/2024-04-24-05-16-32.gh-issue-118209.Ryyzlz.rst new file mode 100644 index 00000000000000..da70b2528919e1 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-04-24-05-16-32.gh-issue-118209.Ryyzlz.rst @@ -0,0 +1,2 @@ +Avoid crashing in :mod:`mmap` on Windows when the mapped memory is inaccessible +due to file system errors or access violations. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index dfc16ff4370349..99a85e9e49ad47 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -41,6 +41,7 @@ #ifdef MS_WINDOWS #include +#include // LsaNtStatusToWinError static int my_getpagesize(void) { @@ -255,6 +256,208 @@ do { \ } while (0) #endif /* UNIX */ +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +static DWORD +filter_page_exception(EXCEPTION_POINTERS *ptrs, EXCEPTION_RECORD *record) +{ + *record = *ptrs->ExceptionRecord; + if (record->ExceptionCode == EXCEPTION_IN_PAGE_ERROR || + record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) + { + return EXCEPTION_EXECUTE_HANDLER; + } + return EXCEPTION_CONTINUE_SEARCH; +} + +static DWORD +filter_page_exception_method(mmap_object *self, EXCEPTION_POINTERS *ptrs, + EXCEPTION_RECORD *record) +{ + *record = *ptrs->ExceptionRecord; + if (record->ExceptionCode == EXCEPTION_IN_PAGE_ERROR || + record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) + { + + ULONG_PTR address = record->ExceptionInformation[1]; + if (address >= (ULONG_PTR) self->data && + address < (ULONG_PTR) self->data + (ULONG_PTR) self->size) + { + return EXCEPTION_EXECUTE_HANDLER; + } + } + return EXCEPTION_CONTINUE_SEARCH; +} +#endif + +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +#define HANDLE_INVALID_MEM(sourcecode) \ +do { \ + EXCEPTION_RECORD record; \ + __try { \ + sourcecode \ + } \ + __except (filter_page_exception(GetExceptionInformation(), &record)) { \ + assert(record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR || \ + record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION); \ + if (record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR) { \ + NTSTATUS status = (NTSTATUS) record.ExceptionInformation[2]; \ + ULONG code = LsaNtStatusToWinError(status); \ + PyErr_SetFromWindowsErr(code); \ + } \ + else if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { \ + PyErr_SetFromWindowsErr(ERROR_NOACCESS); \ + } \ + return -1; \ + } \ +} while (0) +#else +#define HANDLE_INVALID_MEM(sourcecode) \ +do { \ + sourcecode \ +} while (0) +#endif + +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +#define HANDLE_INVALID_MEM_METHOD(self, sourcecode) \ +do { \ + EXCEPTION_RECORD record; \ + __try { \ + sourcecode \ + } \ + __except (filter_page_exception_method(self, GetExceptionInformation(), \ + &record)) { \ + assert(record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR || \ + record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION); \ + if (record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR) { \ + NTSTATUS status = (NTSTATUS) record.ExceptionInformation[2]; \ + ULONG code = LsaNtStatusToWinError(status); \ + PyErr_SetFromWindowsErr(code); \ + } \ + else if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { \ + PyErr_SetFromWindowsErr(ERROR_NOACCESS); \ + } \ + return -1; \ + } \ +} while (0) +#else +#define HANDLE_INVALID_MEM_METHOD(self, sourcecode) \ +do { \ + sourcecode \ +} while (0) +#endif + +int +safe_memcpy(void *dest, const void *src, size_t count) +{ + HANDLE_INVALID_MEM( + memcpy(dest, src, count); + ); + return 0; +} + +int +safe_byte_copy(char *dest, const char *src) +{ + HANDLE_INVALID_MEM( + *dest = *src; + ); + return 0; +} + +int +safe_memchr(char **out, const void *ptr, int ch, size_t count) +{ + HANDLE_INVALID_MEM( + *out = (char *) memchr(ptr, ch, count); + ); + return 0; +} + +int +safe_memmove(void *dest, const void *src, size_t count) +{ + HANDLE_INVALID_MEM( + memmove(dest, src, count); + ); + return 0; +} + +int +safe_copy_from_slice(char *dest, const char *src, Py_ssize_t start, + Py_ssize_t step, Py_ssize_t slicelen) +{ + HANDLE_INVALID_MEM( + size_t cur; + Py_ssize_t i; + for (cur = start, i = 0; i < slicelen; cur += step, i++) { + dest[cur] = src[i]; + } + ); + return 0; +} + +int +safe_copy_to_slice(char *dest, const char *src, Py_ssize_t start, + Py_ssize_t step, Py_ssize_t slicelen) +{ + HANDLE_INVALID_MEM( + size_t cur; + Py_ssize_t i; + for (cur = start, i = 0; i < slicelen; cur += step, i++) { + dest[i] = src[cur]; + } + ); + return 0; +} + + +int +_safe_PyBytes_Find(Py_ssize_t *out, mmap_object *self, const char *haystack, + Py_ssize_t len_haystack, const char *needle, + Py_ssize_t len_needle, Py_ssize_t offset) +{ + HANDLE_INVALID_MEM_METHOD(self, + *out = _PyBytes_Find(haystack, len_haystack, needle, len_needle, offset); + ); + return 0; +} + +int +_safe_PyBytes_ReverseFind(Py_ssize_t *out, mmap_object *self, + const char *haystack, Py_ssize_t len_haystack, + const char *needle, Py_ssize_t len_needle, + Py_ssize_t offset) +{ + HANDLE_INVALID_MEM_METHOD(self, + *out = _PyBytes_ReverseFind(haystack, len_haystack, needle, len_needle, + offset); + ); + return 0; +} + +PyObject * +_safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) { + if (num_bytes == 1) { + char dest; + if (safe_byte_copy(&dest, start) < 0) { + return NULL; + } + else { + return PyBytes_FromStringAndSize(&dest, 1); + } + } + else { + PyObject *result = PyBytes_FromStringAndSize(NULL, num_bytes); + if (result == NULL) { + return NULL; + } + if (safe_memcpy(PyBytes_AS_STRING(result), start, num_bytes) < 0) { + Py_CLEAR(result); + } + return result; + } +} + static PyObject * mmap_read_byte_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) @@ -264,7 +467,12 @@ mmap_read_byte_method(mmap_object *self, PyErr_SetString(PyExc_ValueError, "read byte out of range"); return NULL; } - return PyLong_FromLong((unsigned char)self->data[self->pos++]); + char dest; + if (safe_byte_copy(&dest, self->data + self->pos) < 0) { + return NULL; + } + self->pos++; + return PyLong_FromLong((unsigned char) dest); } static PyObject * @@ -273,7 +481,6 @@ mmap_read_line_method(mmap_object *self, { Py_ssize_t remaining; char *start, *eol; - PyObject *result; CHECK_VALID(NULL); @@ -281,13 +488,20 @@ mmap_read_line_method(mmap_object *self, if (!remaining) return PyBytes_FromString(""); start = self->data + self->pos; - eol = memchr(start, '\n', remaining); + + if (safe_memchr(&eol, start, '\n', remaining) < 0) { + return NULL; + } + if (!eol) eol = self->data + self->size; else ++eol; /* advance past newline */ - result = PyBytes_FromStringAndSize(start, (eol - start)); - self->pos += (eol - start); + + PyObject *result = _safe_PyBytes_FromStringAndSize(start, eol - start); + if (result != NULL) { + self->pos += (eol - start); + } return result; } @@ -296,7 +510,6 @@ mmap_read_method(mmap_object *self, PyObject *args) { Py_ssize_t num_bytes = PY_SSIZE_T_MAX, remaining; - PyObject *result; CHECK_VALID(NULL); if (!PyArg_ParseTuple(args, "|O&:read", _Py_convert_optional_to_ssize_t, &num_bytes)) @@ -307,8 +520,12 @@ mmap_read_method(mmap_object *self, remaining = (self->pos < self->size) ? self->size - self->pos : 0; if (num_bytes < 0 || num_bytes > remaining) num_bytes = remaining; - result = PyBytes_FromStringAndSize(&self->data[self->pos], num_bytes); - self->pos += num_bytes; + + PyObject *result = _safe_PyBytes_FromStringAndSize(self->data + self->pos, + num_bytes); + if (result != NULL) { + self->pos += num_bytes; + } return result; } @@ -341,25 +558,38 @@ mmap_gfind(mmap_object *self, else if (end > self->size) end = self->size; - Py_ssize_t res; + Py_ssize_t index; + PyObject *result; CHECK_VALID_OR_RELEASE(NULL, view); if (end < start) { - res = -1; + result = PyLong_FromSsize_t(-1); } else if (reverse) { assert(0 <= start && start <= end && end <= self->size); - res = _PyBytes_ReverseFind( + if (_safe_PyBytes_ReverseFind(&index, self, self->data + start, end - start, - view.buf, view.len, start); + view.buf, view.len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } } else { assert(0 <= start && start <= end && end <= self->size); - res = _PyBytes_Find( + if (_safe_PyBytes_Find(&index, self, self->data + start, end - start, - view.buf, view.len, start); + view.buf, view.len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } } PyBuffer_Release(&view); - return PyLong_FromSsize_t(res); + return result; } } @@ -432,10 +662,16 @@ mmap_write_method(mmap_object *self, } CHECK_VALID_OR_RELEASE(NULL, data); - memcpy(&self->data[self->pos], data.buf, data.len); - self->pos += data.len; + PyObject *result; + if (safe_memcpy(self->data + self->pos, data.buf, data.len) < 0) { + result = NULL; + } + else { + self->pos += data.len; + result = PyLong_FromSsize_t(data.len); + } PyBuffer_Release(&data); - return PyLong_FromSsize_t(data.len); + return result; } static PyObject * @@ -452,14 +688,16 @@ mmap_write_byte_method(mmap_object *self, return NULL; CHECK_VALID(NULL); - if (self->pos < self->size) { - self->data[self->pos++] = value; - Py_RETURN_NONE; - } - else { + if (self->pos >= self->size) { PyErr_SetString(PyExc_ValueError, "write byte out of range"); return NULL; } + + if (safe_byte_copy(self->data + self->pos, &value) < 0) { + return NULL; + } + self->pos++; + Py_RETURN_NONE; } static PyObject * @@ -763,8 +1001,9 @@ mmap_move_method(mmap_object *self, PyObject *args) goto bounds; CHECK_VALID(NULL); - memmove(&self->data[dest], &self->data[src], cnt); - + if (safe_memmove(self->data + dest, self->data + src, cnt) < 0) { + return NULL; + }; Py_RETURN_NONE; bounds: @@ -855,6 +1094,29 @@ mmap__sizeof__method(mmap_object *self, void *Py_UNUSED(ignored)) } #endif +#if defined(MS_WINDOWS) && defined(Py_DEBUG) +static PyObject * +mmap_protect_method(mmap_object *self, PyObject *args) { + DWORD flNewProtect, flOldProtect; + Py_ssize_t start, length; + + CHECK_VALID(NULL); + + if (!PyArg_ParseTuple(args, "Inn:protect", &flNewProtect, &start, &length)) { + return NULL; + } + + if (!VirtualProtect((void *) (self->data + start), length, flNewProtect, + &flOldProtect)) + { + PyErr_SetFromWindowsErr(GetLastError()); + return NULL; + } + + Py_RETURN_NONE; +} +#endif + #ifdef HAVE_MADVISE static PyObject * mmap_madvise_method(mmap_object *self, PyObject *args) @@ -924,7 +1186,10 @@ static struct PyMethodDef mmap_object_methods[] = { {"__exit__", (PyCFunction) mmap__exit__method, METH_VARARGS}, #ifdef MS_WINDOWS {"__sizeof__", (PyCFunction) mmap__sizeof__method, METH_NOARGS}, -#endif +#ifdef Py_DEBUG + {"_protect", (PyCFunction) mmap_protect_method, METH_VARARGS}, +#endif // Py_DEBUG +#endif // MS_WINDOWS {NULL, NULL} /* sentinel */ }; @@ -968,7 +1233,12 @@ mmap_item(mmap_object *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "mmap index out of range"); return NULL; } - return PyBytes_FromStringAndSize(self->data + i, 1); + + char dest; + if (safe_byte_copy(&dest, self->data + i) < 0) { + return NULL; + } + return PyBytes_FromStringAndSize(&dest, 1); } static PyObject * @@ -987,7 +1257,12 @@ mmap_subscript(mmap_object *self, PyObject *item) return NULL; } CHECK_VALID(NULL); - return PyLong_FromLong(Py_CHARMASK(self->data[i])); + + char dest; + if (safe_byte_copy(&dest, self->data + i) < 0) { + return NULL; + } + return PyLong_FromLong(Py_CHARMASK(dest)); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; @@ -1001,23 +1276,22 @@ mmap_subscript(mmap_object *self, PyObject *item) if (slicelen <= 0) return PyBytes_FromStringAndSize("", 0); else if (step == 1) - return PyBytes_FromStringAndSize(self->data + start, - slicelen); + return _safe_PyBytes_FromStringAndSize(self->data + start, slicelen); else { char *result_buf = (char *)PyMem_Malloc(slicelen); - size_t cur; - Py_ssize_t i; PyObject *result; if (result_buf == NULL) return PyErr_NoMemory(); - for (cur = start, i = 0; i < slicelen; - cur += step, i++) { - result_buf[i] = self->data[cur]; + if (safe_copy_to_slice(result_buf, self->data, start, step, + slicelen) < 0) + { + result = NULL; + } + else { + result = PyBytes_FromStringAndSize(result_buf, slicelen); } - result = PyBytes_FromStringAndSize(result_buf, - slicelen); PyMem_Free(result_buf); return result; } @@ -1052,7 +1326,10 @@ mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v) if (!is_writable(self)) return -1; buf = PyBytes_AsString(v); - self->data[i] = buf[0]; + + if (safe_byte_copy(self->data + i, buf) < 0) { + return -1; + } return 0; } @@ -1097,7 +1374,11 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) return -1; } CHECK_VALID(-1); - self->data[i] = (char) v; + + char v_char = (char) v; + if (safe_byte_copy(self->data + i, &v_char) < 0) { + return -1; + } return 0; } else if (PySlice_Check(item)) { @@ -1123,24 +1404,23 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) } CHECK_VALID_OR_RELEASE(-1, vbuf); + int result = 0; if (slicelen == 0) { } else if (step == 1) { - memcpy(self->data + start, vbuf.buf, slicelen); + if (safe_memcpy(self->data + start, vbuf.buf, slicelen) < 0) { + result = -1; + } } else { - size_t cur; - Py_ssize_t i; - - for (cur = start, i = 0; - i < slicelen; - cur += step, i++) + if (safe_copy_from_slice(self->data, (char *)vbuf.buf, start, step, + slicelen) < 0) { - self->data[cur] = ((char *)vbuf.buf)[i]; + result = -1; } } PyBuffer_Release(&vbuf); - return 0; + return result; } else { PyErr_SetString(PyExc_TypeError, From 7cc5e81a8259fb2e78ed12cbb15ad9e1710f20ed Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 May 2024 12:04:16 +0200 Subject: [PATCH 036/903] gh-117873: Use positional-only parameters in _posixshmem (#118012) * shm_unlink() parameter becomes positional-only. * shm_open() first parameter (path) becomes positional-only, the two following parameters remain positional-or-keyword. --- .../_multiprocessing/clinic/posixshmem.c.h | 19 ++++++++++--------- Modules/_multiprocessing/posixshmem.c | 6 ++++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 1b894ea4c67adc..8151f2e0b07082 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -5,7 +5,7 @@ preserve #if defined(HAVE_SHM_OPEN) PyDoc_STRVAR(_posixshmem_shm_open__doc__, -"shm_open($module, /, path, flags, mode=511)\n" +"shm_open($module, path, /, flags, mode=511)\n" "--\n" "\n" "Open a shared memory object. Returns a file descriptor (integer)."); @@ -21,7 +21,7 @@ static PyObject * _posixshmem_shm_open(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - static char *_keywords[] = {"path", "flags", "mode", NULL}; + static char *_keywords[] = {"", "flags", "mode", NULL}; PyObject *path; int flags; int mode = 511; @@ -45,7 +45,7 @@ _posixshmem_shm_open(PyObject *module, PyObject *args, PyObject *kwargs) #if defined(HAVE_SHM_UNLINK) PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, -"shm_unlink($module, /, path)\n" +"shm_unlink($module, path, /)\n" "--\n" "\n" "Remove a shared memory object (similar to unlink()).\n" @@ -55,21 +55,22 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, "region."); #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ - {"shm_unlink", (PyCFunction)(void(*)(void))_posixshmem_shm_unlink, METH_VARARGS|METH_KEYWORDS, _posixshmem_shm_unlink__doc__}, + {"shm_unlink", (PyCFunction)_posixshmem_shm_unlink, METH_O, _posixshmem_shm_unlink__doc__}, static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path); static PyObject * -_posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) +_posixshmem_shm_unlink(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - static char *_keywords[] = {"path", NULL}; PyObject *path; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "U:shm_unlink", _keywords, - &path)) + if (!PyUnicode_Check(arg)) { + PyErr_Format(PyExc_TypeError, "shm_unlink() argument must be str, not %T", arg); goto exit; + } + path = arg; return_value = _posixshmem_shm_unlink_impl(module, path); exit: @@ -85,4 +86,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=be0661dbed83ea23 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=649877fc45a65129 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index aeb2d79de6f9ed..cc157800ade3c4 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -32,6 +32,7 @@ module _posixshmem /*[clinic input] _posixshmem.shm_open -> int path: unicode + / flags: int mode: int = 0o777 @@ -44,7 +45,7 @@ Open a shared memory object. Returns a file descriptor (integer). static int _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, int mode) -/*[clinic end generated code: output=8d110171a4fa20df input=e83b58fa802fac25]*/ +/*[clinic end generated code: output=8d110171a4fa20df input=0585935e1d3c8050]*/ { int fd; int async_err = 0; @@ -77,6 +78,7 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, /*[clinic input] _posixshmem.shm_unlink path: unicode + / Remove a shared memory object (similar to unlink()). @@ -88,7 +90,7 @@ region. static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=42f8b23d134b9ff5 input=8dc0f87143e3b300]*/ +/*[clinic end generated code: output=42f8b23d134b9ff5 input=298369d013dcad63]*/ { int rv; int async_err = 0; From 7ac933e2609b2ef9b08ccf9c815b682b0e1ede2a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 10 May 2024 19:24:02 +0900 Subject: [PATCH 037/903] gh-118689: Doc: fix ePub build (#118690) --- Doc/conf.py | 4 ++++ Doc/library/allos.rst | 1 - Doc/tools/extensions/glossary_search.py | 2 +- Doc/tools/templates/layout.html | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index 86371d17ae742a..0e86de837d35d2 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -435,6 +435,10 @@ epub_author = 'Python Documentation Authors' epub_publisher = 'Python Software Foundation' +# index pages are not valid xhtml +# https://github.com/sphinx-doc/sphinx/issues/12359 +epub_use_index = False + # Options for the coverage checker # -------------------------------- diff --git a/Doc/library/allos.rst b/Doc/library/allos.rst index f7105d8af8e28b..0223c1054ea5d8 100644 --- a/Doc/library/allos.rst +++ b/Doc/library/allos.rst @@ -16,7 +16,6 @@ but they are available on most other systems as well. Here's an overview: io.rst time.rst argparse.rst - getopt.rst logging.rst logging.config.rst logging.handlers.rst diff --git a/Doc/tools/extensions/glossary_search.py b/Doc/tools/extensions/glossary_search.py index 59a6862ea3d3f4..232782093926f6 100644 --- a/Doc/tools/extensions/glossary_search.py +++ b/Doc/tools/extensions/glossary_search.py @@ -20,7 +20,7 @@ def process_glossary_nodes(app, doctree, fromdocname): - if app.builder.format != 'html': + if app.builder.format != 'html' or app.builder.embedded: return terms = {} diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 3c12b01b558f83..e931147813ae03 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -42,6 +42,7 @@ {{ super() }} +{%- if not embedded %} +{%- endif %} {% endblock %} From 004db2170ecfc27fc8ceea29fee0a10c1b7dafdf Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 10 May 2024 15:46:20 +0300 Subject: [PATCH 038/903] Rename `notimplemented_methods` into `nodefault_methods` (#118896) --- Objects/typevarobject.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index cc916045266aea..c8ab14053de418 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -73,7 +73,7 @@ NoDefault_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) return PyUnicode_FromString("NoDefault"); } -static PyMethodDef notimplemented_methods[] = { +static PyMethodDef nodefault_methods[] = { {"__reduce__", NoDefault_reduce, METH_NOARGS, NULL}, {NULL, NULL} }; @@ -98,7 +98,7 @@ nodefault_dealloc(PyObject *nodefault) _Py_SetImmortal(nodefault); } -PyDoc_STRVAR(notimplemented_doc, +PyDoc_STRVAR(nodefault_doc, "NoDefaultType()\n" "--\n\n" "The type of the NoDefault singleton."); @@ -109,8 +109,8 @@ PyTypeObject _PyNoDefault_Type = { .tp_dealloc = nodefault_dealloc, .tp_repr = NoDefault_repr, .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = notimplemented_doc, - .tp_methods = notimplemented_methods, + .tp_doc = nodefault_doc, + .tp_methods = nodefault_methods, .tp_new = nodefault_new, }; From a895756aec688c049a983199e2d7fb801d6502c8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 May 2024 15:24:06 +0200 Subject: [PATCH 039/903] gh-117398: Move types to datetime state (#118606) Move types to the datetime_state structure of the _datetime extension. --- Modules/_datetimemodule.c | 176 +++++++++++++++++++++----------------- 1 file changed, 99 insertions(+), 77 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 8164715a66ff09..9a66f0358179b5 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -25,24 +25,15 @@ # include /* struct timeval */ #endif -#define PyDate_Check(op) PyObject_TypeCheck(op, &PyDateTime_DateType) -#define PyDate_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DateType) - -#define PyDateTime_Check(op) PyObject_TypeCheck(op, &PyDateTime_DateTimeType) -#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DateTimeType) - -#define PyTime_Check(op) PyObject_TypeCheck(op, &PyDateTime_TimeType) -#define PyTime_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_TimeType) - -#define PyDelta_Check(op) PyObject_TypeCheck(op, &PyDateTime_DeltaType) -#define PyDelta_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DeltaType) - -#define PyTZInfo_Check(op) PyObject_TypeCheck(op, &PyDateTime_TZInfoType) -#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_TZInfoType) - -#define PyTimezone_Check(op) PyObject_TypeCheck(op, &PyDateTime_TimeZoneType) - typedef struct { + PyTypeObject *date_type; + PyTypeObject *datetime_type; + PyTypeObject *delta_type; + PyTypeObject *isocalendar_date_type; + PyTypeObject *time_type; + PyTypeObject *tzinfo_type; + PyTypeObject *timezone_type; + /* Conversion factors. */ PyObject *us_per_ms; // 1_000 PyObject *us_per_second; // 1_000_000 @@ -61,7 +52,27 @@ typedef struct { static datetime_state _datetime_global_state; -#define STATIC_STATE() (&_datetime_global_state) +static inline datetime_state* get_datetime_state(void) +{ + return &_datetime_global_state; +} + +#define PyDate_Check(op) PyObject_TypeCheck(op, get_datetime_state()->date_type) +#define PyDate_CheckExact(op) Py_IS_TYPE(op, get_datetime_state()->date_type) + +#define PyDateTime_Check(op) PyObject_TypeCheck(op, get_datetime_state()->datetime_type) +#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, get_datetime_state()->datetime_type) + +#define PyTime_Check(op) PyObject_TypeCheck(op, get_datetime_state()->time_type) +#define PyTime_CheckExact(op) Py_IS_TYPE(op, get_datetime_state()->time_type) + +#define PyDelta_Check(op) PyObject_TypeCheck(op, get_datetime_state()->delta_type) +#define PyDelta_CheckExact(op) Py_IS_TYPE(op, get_datetime_state()->delta_type) + +#define PyTZInfo_Check(op) PyObject_TypeCheck(op, get_datetime_state()->tzinfo_type) +#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, get_datetime_state()->tzinfo_type) + +#define PyTimezone_Check(op) PyObject_TypeCheck(op, get_datetime_state()->timezone_type) /* We require that C int be at least 32 bits, and use int virtually * everywhere. In just a few cases we use a temp long, where a Python @@ -142,25 +153,16 @@ static datetime_state _datetime_global_state; */ #define MONTH_IS_SANE(M) ((unsigned int)(M) - 1 < 12) -/* Forward declarations. */ -static PyTypeObject PyDateTime_DateType; -static PyTypeObject PyDateTime_DateTimeType; -static PyTypeObject PyDateTime_DeltaType; -static PyTypeObject PyDateTime_IsoCalendarDateType; -static PyTypeObject PyDateTime_TimeType; -static PyTypeObject PyDateTime_TZInfoType; -static PyTypeObject PyDateTime_TimeZoneType; - static int check_tzinfo_subclass(PyObject *p); /*[clinic input] module datetime -class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType" -class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType" -class datetime.time "PyDateTime_Time *" "&PyDateTime_TimeType" -class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType" +class datetime.datetime "PyDateTime_DateTime *" "get_datetime_state()->datetime_type" +class datetime.date "PyDateTime_Date *" "get_datetime_state()->date_type" +class datetime.time "PyDateTime_Time *" "get_datetime_state()->time_type" +class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "get_datetime_state()->isocalendar_date_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6f65a48dd22fa40f]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c8f3d834a860d50a]*/ #include "clinic/_datetimemodule.c.h" @@ -979,7 +981,7 @@ new_date_ex(int year, int month, int day, PyTypeObject *type) } #define new_date(year, month, day) \ - new_date_ex(year, month, day, &PyDateTime_DateType) + new_date_ex(year, month, day, get_datetime_state()->date_type) // Forward declaration static PyObject * @@ -989,12 +991,13 @@ new_datetime_ex(int, int, int, int, int, int, int, PyObject *, PyTypeObject *); static PyObject * new_date_subclass_ex(int year, int month, int day, PyObject *cls) { + datetime_state *st = get_datetime_state(); PyObject *result; // We have "fast path" constructors for two subclasses: date and datetime - if ((PyTypeObject *)cls == &PyDateTime_DateType) { + if ((PyTypeObject *)cls == st->date_type) { result = new_date_ex(year, month, day, (PyTypeObject *)cls); } - else if ((PyTypeObject *)cls == &PyDateTime_DateTimeType) { + else if ((PyTypeObject *)cls == st->datetime_type) { result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, (PyTypeObject *)cls); } @@ -1049,7 +1052,7 @@ new_datetime_ex(int year, int month, int day, int hour, int minute, #define new_datetime(y, m, d, hh, mm, ss, us, tzinfo, fold) \ new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, \ - &PyDateTime_DateTimeType) + get_datetime_state()->datetime_type) static PyObject * call_subclass_fold(PyObject *cls, int fold, const char *format, ...) @@ -1088,9 +1091,11 @@ call_subclass_fold(PyObject *cls, int fold, const char *format, ...) static PyObject * new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, - int fold, PyObject *cls) { + int fold, PyObject *cls) +{ + datetime_state *st = get_datetime_state(); PyObject* dt; - if ((PyTypeObject*)cls == &PyDateTime_DateTimeType) { + if ((PyTypeObject*)cls == st->datetime_type) { // Use the fast path constructor dt = new_datetime(year, month, day, hour, minute, second, usecond, tzinfo, fold); @@ -1152,14 +1157,15 @@ new_time_ex(int hour, int minute, int second, int usecond, } #define new_time(hh, mm, ss, us, tzinfo, fold) \ - new_time_ex2(hh, mm, ss, us, tzinfo, fold, &PyDateTime_TimeType) + new_time_ex2(hh, mm, ss, us, tzinfo, fold, get_datetime_state()->time_type) static PyObject * new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, PyObject *tzinfo, int fold, PyObject *cls) { PyObject *t; - if ((PyTypeObject*)cls == &PyDateTime_TimeType) { + datetime_state *st = get_datetime_state(); + if ((PyTypeObject*)cls == st->time_type) { // Use the fast path constructor t = new_time(hour, minute, second, usecond, tzinfo, fold); } @@ -1203,7 +1209,7 @@ new_delta_ex(int days, int seconds, int microseconds, int normalize, } #define new_delta(d, s, us, normalize) \ - new_delta_ex(d, s, us, normalize, &PyDateTime_DeltaType) + new_delta_ex(d, s, us, normalize, get_datetime_state()->delta_type) typedef struct @@ -1221,7 +1227,8 @@ static PyObject * create_timezone(PyObject *offset, PyObject *name) { PyDateTime_TimeZone *self; - PyTypeObject *type = &PyDateTime_TimeZoneType; + datetime_state *st = get_datetime_state(); + PyTypeObject *type = st->timezone_type; assert(offset != NULL); assert(PyDelta_Check(offset)); @@ -1246,7 +1253,7 @@ new_timezone(PyObject *offset, PyObject *name) assert(name == NULL || PyUnicode_Check(name)); if (name == NULL && delta_bool((PyDateTime_Delta *)offset) == 0) { - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); return Py_NewRef(st->utc); } if ((GET_TD_DAYS(offset) == -1 && @@ -1460,7 +1467,7 @@ tzinfo_from_isoformat_results(int rv, int tzoffset, int tz_useconds) if (rv == 1) { // Create a timezone from offset in seconds (0 returns UTC) if (tzoffset == 0) { - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); return Py_NewRef(st->utc); } @@ -1893,7 +1900,7 @@ delta_to_microseconds(PyDateTime_Delta *self) x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) goto Done; - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); x2 = PyNumber_Multiply(x1, st->seconds_per_day); /* days in seconds */ if (x2 == NULL) goto Done; @@ -1966,7 +1973,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); tuple = checked_divmod(pyus, st->us_per_second); if (tuple == NULL) { goto Done; @@ -2019,7 +2026,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) } #define microseconds_to_delta(pymicros) \ - microseconds_to_delta_ex(pymicros, &PyDateTime_DeltaType) + microseconds_to_delta_ex(pymicros, get_datetime_state()->delta_type) static PyObject * multiply_int_timedelta(PyObject *intobj, PyDateTime_Delta *delta) @@ -2585,7 +2592,7 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) y = accum("microseconds", x, us, _PyLong_GetOne(), &leftover_us); CLEANUP; } - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); if (ms) { y = accum("milliseconds", x, ms, st->us_per_ms, &leftover_us); CLEANUP; @@ -2762,7 +2769,7 @@ delta_total_seconds(PyObject *self, PyObject *Py_UNUSED(ignored)) if (total_microseconds == NULL) return NULL; - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); total_seconds = PyNumber_TrueDivide(total_microseconds, st->us_per_second); Py_DECREF(total_microseconds); @@ -3475,8 +3482,9 @@ date_isocalendar(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) week = 0; } - PyObject* v = iso_calendar_date_new_impl(&PyDateTime_IsoCalendarDateType, - year, week + 1, day + 1); + datetime_state *st = get_datetime_state(); + PyObject *v = iso_calendar_date_new_impl(st->isocalendar_date_type, + year, week + 1, day + 1); if (v == NULL) { return NULL; } @@ -3945,8 +3953,9 @@ timezone_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *offset; PyObject *name = NULL; + datetime_state *st = get_datetime_state(); if (PyArg_ParseTupleAndKeywords(args, kw, "O!|U:timezone", timezone_kws, - &PyDateTime_DeltaType, &offset, &name)) + st->delta_type, &offset, &name)) return new_timezone(offset, name); return NULL; @@ -3999,7 +4008,7 @@ timezone_repr(PyDateTime_TimeZone *self) to use Py_TYPE(self)->tp_name here. */ const char *type_name = Py_TYPE(self)->tp_name; - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); if (((PyObject *)self) == st->utc) { return PyUnicode_FromFormat("%s.utc", type_name); } @@ -4022,7 +4031,7 @@ timezone_str(PyDateTime_TimeZone *self) if (self->name != NULL) { return Py_NewRef(self->name); } - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); if ((PyObject *)self == st->utc || (GET_TD_DAYS(self->offset) == 0 && GET_TD_SECONDS(self->offset) == 0 && @@ -4686,7 +4695,8 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { } PyObject *t; - if ( (PyTypeObject *)cls == &PyDateTime_TimeType ) { + datetime_state *st = get_datetime_state(); + if ( (PyTypeObject *)cls == st->time_type) { t = new_time(hour, minute, second, microsecond, tzinfo, 0); } else { t = PyObject_CallFunction(cls, "iiiiO", @@ -5284,9 +5294,10 @@ datetime_combine(PyObject *cls, PyObject *args, PyObject *kw) PyObject *tzinfo = NULL; PyObject *result = NULL; + datetime_state *st = get_datetime_state(); if (PyArg_ParseTupleAndKeywords(args, kw, "O!O!|O:combine", keywords, - &PyDateTime_DateType, &date, - &PyDateTime_TimeType, &time, &tzinfo)) { + st->date_type, &date, + st->time_type, &time, &tzinfo)) { if (tzinfo == NULL) { if (HASTZINFO(time)) tzinfo = ((PyDateTime_Time *)time)->tzinfo; @@ -6116,6 +6127,7 @@ local_timezone_from_timestamp(time_t timestamp) delta = new_delta(0, local_time_tm.tm_gmtoff, 0, 1); #else /* HAVE_STRUCT_TM_TM_ZONE */ { + datetime_state *st = get_datetime_state(); PyObject *local_time, *utc_time; struct tm utc_time_tm; char buf[100]; @@ -6170,10 +6182,11 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); delta = datetime_subtract((PyObject *)utc_time, st->epoch); if (delta == NULL) return NULL; + one_second = new_delta(0, 1, 0, 0); if (one_second == NULL) { Py_DECREF(delta); @@ -6283,7 +6296,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) if (result == NULL) return NULL; - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); /* Make sure result is aware and UTC. */ if (!HASTZINFO(result)) { temp = (PyObject *)result; @@ -6408,7 +6421,7 @@ datetime_timestamp(PyDateTime_DateTime *self, PyObject *Py_UNUSED(ignored)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); PyObject *delta; delta = datetime_subtract((PyObject *)self, st->epoch); if (delta == NULL) @@ -6712,16 +6725,18 @@ static PyMethodDef module_methods[] = { static inline PyDateTime_CAPI * get_datetime_capi(void) { + datetime_state *st = get_datetime_state(); + PyDateTime_CAPI *capi = PyMem_Malloc(sizeof(PyDateTime_CAPI)); if (capi == NULL) { PyErr_NoMemory(); return NULL; } - capi->DateType = &PyDateTime_DateType; - capi->DateTimeType = &PyDateTime_DateTimeType; - capi->TimeType = &PyDateTime_TimeType; - capi->DeltaType = &PyDateTime_DeltaType; - capi->TZInfoType = &PyDateTime_TZInfoType; + capi->DateType = st->date_type; + capi->DateTimeType = st->datetime_type; + capi->TimeType = st->time_type; + capi->DeltaType = st->delta_type; + capi->TZInfoType = st->tzinfo_type; capi->Date_FromDate = new_date_ex; capi->DateTime_FromDateAndTime = new_datetime_ex; capi->Time_FromTime = new_time_ex; @@ -6733,7 +6748,6 @@ get_datetime_capi(void) capi->Time_FromTimeAndFold = new_time_ex2; // Make sure this function is called after utc has // been initialized. - datetime_state *st = STATIC_STATE(); assert(st->utc != NULL); capi->TimeZone_UTC = st->utc; // borrowed ref return capi; @@ -6749,7 +6763,7 @@ datetime_destructor(PyObject *op) static int datetime_clear(PyObject *module) { - datetime_state *st = STATIC_STATE(); + datetime_state *st = get_datetime_state(); Py_CLEAR(st->us_per_ms); Py_CLEAR(st->us_per_second); @@ -6778,6 +6792,14 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) static int init_state(datetime_state *st) { + st->date_type = &PyDateTime_DateType; + st->datetime_type = &PyDateTime_DateTimeType; + st->delta_type = &PyDateTime_DeltaType; + st->isocalendar_date_type = &PyDateTime_IsoCalendarDateType; + st->time_type = &PyDateTime_TimeType; + st->tzinfo_type = &PyDateTime_TZInfoType; + st->timezone_type = &PyDateTime_TimeZoneType; + st->us_per_ms = PyLong_FromLong(1000); if (st->us_per_ms == NULL) { return -1; @@ -6854,6 +6876,11 @@ _datetime_exec(PyObject *module) goto error; } + datetime_state *st = get_datetime_state(); + if (init_state(st) < 0) { + goto error; + } + #define DATETIME_ADD_MACRO(dict, c, value_expr) \ do { \ PyObject *value = (value_expr); \ @@ -6868,39 +6895,34 @@ _datetime_exec(PyObject *module) } while(0) /* timedelta values */ - PyObject *d = PyDateTime_DeltaType.tp_dict; + PyObject *d = st->delta_type->tp_dict; DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); DATETIME_ADD_MACRO(d, "max", new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); /* date values */ - d = PyDateTime_DateType.tp_dict; + d = st->date_type->tp_dict; DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); /* time values */ - d = PyDateTime_TimeType.tp_dict; + d = st->time_type->tp_dict; DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); /* datetime values */ - d = PyDateTime_DateTimeType.tp_dict; + d = st->datetime_type->tp_dict; DATETIME_ADD_MACRO(d, "min", new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, 999999, Py_None, 0)); DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - datetime_state *st = STATIC_STATE(); - if (init_state(st) < 0) { - goto error; - } - /* timezone values */ - d = PyDateTime_TimeZoneType.tp_dict; + d = st->timezone_type->tp_dict; if (PyDict_SetItemString(d, "utc", st->utc) < 0) { goto error; } From db5af7da092409030c9fbe0a3a986bd0ee441b8b Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 10 May 2024 09:54:18 -0400 Subject: [PATCH 040/903] gh-118789: Restore hidden `_PyWeakref_ClearRef` (#118797) _PyWeakref_ClearRef was previously exposed in the public C-API, although it begins with an underscore and is not documented. It's used by a few C-API extensions. There is currently no alternative public API that can replace its use. _PyWeakref_ClearWeakRefsExceptCallbacks is the only thread-safe way to use _PyWeakref_ClearRef in the free-threaded build. This exposes the C symbol, but does not make the API public. --- Include/cpython/weakrefobject.h | 2 ++ Include/internal/pycore_weakref.h | 2 -- .../next/C API/2024-05-08-20-13-00.gh-issue-118789.m88uUa.rst | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-05-08-20-13-00.gh-issue-118789.m88uUa.rst diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index 9a796098c6b48f..dcca166d7357cc 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -40,6 +40,8 @@ struct _PyWeakReference { #endif }; +PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); + Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { PyWeakReference *ref; diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index e057a27340f718..cc6c7ff9a9b438 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -111,8 +111,6 @@ extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyObject *obj); // intact. extern void _PyWeakref_ClearWeakRefsExceptCallbacks(PyObject *obj); -extern void _PyWeakref_ClearRef(PyWeakReference *self); - PyAPI_FUNC(int) _PyWeakref_IsDead(PyObject *weakref); #ifdef __cplusplus diff --git a/Misc/NEWS.d/next/C API/2024-05-08-20-13-00.gh-issue-118789.m88uUa.rst b/Misc/NEWS.d/next/C API/2024-05-08-20-13-00.gh-issue-118789.m88uUa.rst new file mode 100644 index 00000000000000..a2acc16b2c1d01 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-08-20-13-00.gh-issue-118789.m88uUa.rst @@ -0,0 +1,2 @@ +Restore ``_PyWeakref_ClearRef`` that was previously removed in Python 3.13 +alpha 1. From 22d5185308f85efa22ec1e8251c409fe1cbd9e6b Mon Sep 17 00:00:00 2001 From: mpage Date: Fri, 10 May 2024 06:59:14 -0700 Subject: [PATCH 041/903] gh-117657: Fix data races reported by TSAN on `interp->threads.main` (#118865) Use relaxed loads/stores when reading/writing to this field. --- ...-05-09-19-47-12.gh-issue-117657.Vn0Yey.rst | 1 + Python/pystate.c | 31 ++++++++++++------- Tools/tsan/suppressions_free_threading.txt | 2 -- 3 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-09-19-47-12.gh-issue-117657.Vn0Yey.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-09-19-47-12.gh-issue-117657.Vn0Yey.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-09-19-47-12.gh-issue-117657.Vn0Yey.rst new file mode 100644 index 00000000000000..db4c5813ca610c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-09-19-47-12.gh-issue-117657.Vn0Yey.rst @@ -0,0 +1 @@ +Fix data races on the field that stores a pointer to the interpreter's main thread that occur in free-threaded builds. diff --git a/Python/pystate.c b/Python/pystate.c index b1e085bb806915..de6a768a50f997 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1038,6 +1038,17 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) } #endif +static inline void +set_main_thread(PyInterpreterState *interp, PyThreadState *tstate) +{ + _Py_atomic_store_ptr_relaxed(&interp->threads.main, tstate); +} + +static inline PyThreadState * +get_main_thread(PyInterpreterState *interp) +{ + return _Py_atomic_load_ptr_relaxed(&interp->threads.main); +} int _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) @@ -1052,21 +1063,22 @@ _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) "current tstate has wrong interpreter"); return -1; } - interp->threads.main = tstate; + set_main_thread(interp, tstate); + return 0; } void _PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) { - assert(interp->threads.main == current_fast_get()); - interp->threads.main = NULL; + assert(get_main_thread(interp) == current_fast_get()); + set_main_thread(interp, NULL); } int _PyInterpreterState_IsRunningMain(PyInterpreterState *interp) { - if (interp->threads.main != NULL) { + if (get_main_thread(interp) != NULL) { return 1; } // Embedders might not know to call _PyInterpreterState_SetRunningMain(), @@ -1082,18 +1094,15 @@ int _PyThreadState_IsRunningMain(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; - if (interp->threads.main != NULL) { - return tstate == interp->threads.main; - } // See the note in _PyInterpreterState_IsRunningMain() about // possible false negatives here for embedders. - return 0; + return get_main_thread(interp) == tstate; } int _PyInterpreterState_FailIfRunningMain(PyInterpreterState *interp) { - if (interp->threads.main != NULL) { + if (get_main_thread(interp) != NULL) { PyErr_SetString(PyExc_InterpreterError, "interpreter already running"); return -1; @@ -1105,8 +1114,8 @@ void _PyInterpreterState_ReinitRunningMain(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; - if (interp->threads.main != tstate) { - interp->threads.main = NULL; + if (get_main_thread(interp) != tstate) { + set_main_thread(interp, NULL); } } diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 7f91a9113c4e1a..d5f4cd7acd36b7 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -33,8 +33,6 @@ race_top:_mi_heap_delayed_free_partial race_top:_PyEval_EvalFrameDefault race_top:_PyImport_AcquireLock race_top:_PyImport_ReleaseLock -race_top:_PyInterpreterState_SetNotRunningMain -race_top:_PyInterpreterState_IsRunningMain # https://gist.github.com/mpage/0a24eb2dd458441ededb498e9b0e5de8 race_top:_PyParkingLot_Park race_top:_PyType_HasFeature From 33d20199af65c741bdc908a968edd8dc179b6974 Mon Sep 17 00:00:00 2001 From: Alex Turner Date: Fri, 10 May 2024 15:26:35 +0100 Subject: [PATCH 042/903] gh-117657: Fix QSBR race condition (#118843) `_Py_qsbr_unregister` is called when the PyThreadState is already detached, so the access to `tstate->qsbr` isn't safe without locking the shared mutex. Grab the `struct _qsbr_shared` from the interpreter instead. --- Include/internal/pycore_qsbr.h | 2 +- Python/pystate.c | 2 +- Python/qsbr.c | 11 ++++++----- Tools/tsan/suppressions_free_threading.txt | 1 - 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index c3680a205542f7..20e643e172b38d 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -140,7 +140,7 @@ _Py_qsbr_register(struct _PyThreadStateImpl *tstate, // Disassociates a PyThreadState from the QSBR state and frees the QSBR state. extern void -_Py_qsbr_unregister(struct _PyThreadStateImpl *tstate); +_Py_qsbr_unregister(PyThreadState *tstate); extern void _Py_qsbr_fini(PyInterpreterState *interp); diff --git a/Python/pystate.c b/Python/pystate.c index de6a768a50f997..0832b37c278c76 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1794,7 +1794,7 @@ tstate_delete_common(PyThreadState *tstate) HEAD_UNLOCK(runtime); #ifdef Py_GIL_DISABLED - _Py_qsbr_unregister((_PyThreadStateImpl *)tstate); + _Py_qsbr_unregister(tstate); #endif // XXX Unbind in PyThreadState_Clear(), or earlier diff --git a/Python/qsbr.c b/Python/qsbr.c index d7ac8f479cda1b..1e02ff9c2e45f0 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -231,20 +231,21 @@ _Py_qsbr_register(_PyThreadStateImpl *tstate, PyInterpreterState *interp, } void -_Py_qsbr_unregister(_PyThreadStateImpl *tstate) +_Py_qsbr_unregister(PyThreadState *tstate) { - struct _qsbr_shared *shared = tstate->qsbr->shared; + struct _qsbr_shared *shared = &tstate->interp->qsbr; + struct _PyThreadStateImpl *tstate_imp = (_PyThreadStateImpl*) tstate; PyMutex_Lock(&shared->mutex); // NOTE: we must load (or reload) the thread state's qbsr inside the mutex // because the array may have been resized (changing tstate->qsbr) while // we waited to acquire the mutex. - struct _qsbr_thread_state *qsbr = tstate->qsbr; + struct _qsbr_thread_state *qsbr = tstate_imp->qsbr; assert(qsbr->seq == 0 && "thread state must be detached"); - assert(qsbr->allocated && qsbr->tstate == (PyThreadState *)tstate); + assert(qsbr->allocated && qsbr->tstate == tstate); - tstate->qsbr = NULL; + tstate_imp->qsbr = NULL; qsbr->tstate = NULL; qsbr->allocated = false; qsbr->freelist_next = shared->freelist; diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index d5f4cd7acd36b7..dfa4a1fe9ca438 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -38,7 +38,6 @@ race_top:_PyParkingLot_Park race_top:_PyType_HasFeature race_top:assign_version_tag race_top:gc_restore_tid -race_top:initialize_new_array race_top:insertdict race_top:lookup_tp_dict race_top:mi_heap_visit_pages From c444362c6e0b6c01f49c3bee864100f52bd3b640 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Fri, 10 May 2024 15:30:42 +0100 Subject: [PATCH 043/903] Correct the argument names for `secrets.choice` and `secrets.randbelow` in `secrets.rst` (GH-118098) Correct the argument names for `secrets.choice` and `secrets.randbelow` in `secrets.rst`. --- Doc/library/secrets.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index e609948de552bd..1401a925103517 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -42,13 +42,13 @@ randomness that your operating system provides. sources provided by the operating system. See :class:`random.SystemRandom` for additional details. -.. function:: choice(sequence) +.. function:: choice(seq) Return a randomly chosen element from a non-empty sequence. -.. function:: randbelow(n) +.. function:: randbelow(exclusive_upper_bound) - Return a random int in the range [0, *n*). + Return a random int in the range [0, *exclusive_upper_bound*). .. function:: randbits(k) From 13d7cf997bc9c22cf67c42fd799413e8325e0039 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 10 May 2024 08:42:00 -0700 Subject: [PATCH 044/903] gh-118895: Call PyType_Ready() on typing.NoDefault (#118897) --- Include/internal/pycore_typevarobject.h | 1 + Lib/test/test_typing.py | 23 +++++++++++++++++-- ...-05-10-05-24-32.gh-issue-118895.wUm5r2.rst | 2 ++ Modules/_typingmodule.c | 3 +++ 4 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-10-05-24-32.gh-issue-118895.wUm5r2.rst diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h index 80a2daf4efc16a..a368edebd622a1 100644 --- a/Include/internal/pycore_typevarobject.h +++ b/Include/internal/pycore_typevarobject.h @@ -18,6 +18,7 @@ extern int _Py_initialize_generic(PyInterpreterState *); extern void _Py_clear_generic_types(PyInterpreterState *); extern PyTypeObject _PyTypeAlias_Type; +extern PyTypeObject _PyNoDefault_Type; extern PyObject _Py_NoDefaultStruct; #ifdef __cplusplus diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 8dde8aedd2a2b1..81fea41e9b9823 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -45,7 +45,7 @@ import weakref import types -from test.support import captured_stderr, cpython_only, infinite_recursion +from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings from test.typinganndata import ann_module695, mod_generics_cache, _typed_dict_helper @@ -10236,15 +10236,34 @@ def test_pickling(self): def test_constructor(self): self.assertIs(NoDefault, type(NoDefault)()) with self.assertRaises(TypeError): - NoDefault(1) + type(NoDefault)(1) def test_repr(self): self.assertEqual(repr(NoDefault), 'typing.NoDefault') + @requires_docstrings + def test_doc(self): + self.assertIsInstance(NoDefault.__doc__, str) + + def test_class(self): + self.assertIs(NoDefault.__class__, type(NoDefault)) + def test_no_call(self): with self.assertRaises(TypeError): NoDefault() + def test_no_attributes(self): + with self.assertRaises(AttributeError): + NoDefault.foo = 3 + with self.assertRaises(AttributeError): + NoDefault.foo + + # TypeError is consistent with the behavior of NoneType + with self.assertRaises(TypeError): + type(NoDefault).foo = 3 + with self.assertRaises(AttributeError): + type(NoDefault).foo + class AllTests(BaseTestCase): """Tests for __all__.""" diff --git a/Misc/NEWS.d/next/Library/2024-05-10-05-24-32.gh-issue-118895.wUm5r2.rst b/Misc/NEWS.d/next/Library/2024-05-10-05-24-32.gh-issue-118895.wUm5r2.rst new file mode 100644 index 00000000000000..226c8d612a039c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-10-05-24-32.gh-issue-118895.wUm5r2.rst @@ -0,0 +1,2 @@ +Setting attributes on :data:`typing.NoDefault` now raises +:exc:`AttributeError` instead of :exc:`TypeError`. diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 09fbb3c5e8b91d..37af00f3071e1d 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -63,6 +63,9 @@ _typing_exec(PyObject *m) if (PyModule_AddObjectRef(m, "TypeAliasType", (PyObject *)&_PyTypeAlias_Type) < 0) { return -1; } + if (PyType_Ready(&_PyNoDefault_Type) < 0) { + return -1; + } if (PyModule_AddObjectRef(m, "NoDefault", (PyObject *)&_Py_NoDefaultStruct) < 0) { return -1; } From f772d0d08af2beef53db1e076c864cbdf3f5bac9 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 10 May 2024 16:53:46 +0100 Subject: [PATCH 045/903] GH-78707: Drop deprecated `pathlib.PurePath.[is_]relative_to()` arguments (#118780) Remove support for supplying additional positional arguments to `PurePath.relative_to()` and `is_relative_to()`. This has been deprecated since Python 3.12. --- Doc/whatsnew/3.14.rst | 8 ++++++++ Lib/pathlib/_local.py | 20 ++++--------------- Lib/test/test_pathlib/test_pathlib.py | 13 ------------ ...4-05-08-18-59-19.gh-issue-78707._Lz1sw.rst | 3 +++ 4 files changed, 15 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-08-18-59-19.gh-issue-78707._Lz1sw.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 25c43dc0387eaf..52a24d1a9295a3 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -114,6 +114,14 @@ email * The *isdst* parameter has been removed from :func:`email.utils.localtime`. (Contributed by Hugo van Kemenade in :gh:`118798`.) +pathlib +------- + +* Remove support for passing additional positional arguments to + :meth:`pathlib.PurePath.relative_to` and + :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such + arguments are joined onto *other*. + Others ------ diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index b1e678aceb9ce8..f2c627319d520f 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -362,7 +362,7 @@ def with_name(self, name): tail[-1] = name return self._from_parsed_parts(self.drive, self.root, tail) - def relative_to(self, other, /, *_deprecated, walk_up=False): + def relative_to(self, other, *, walk_up=False): """Return the relative path to another path identified by the passed arguments. If the operation is not possible (because this is not related to the other path), raise ValueError. @@ -370,13 +370,7 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): The *walk_up* parameter controls whether `..` may be used to resolve the path. """ - if _deprecated: - msg = ("support for supplying more than one positional argument " - "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): + if not isinstance(other, PurePath): other = self.with_segments(other) for step, path in enumerate(chain([other], other.parents)): if path == self or path in self.parents: @@ -390,16 +384,10 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): parts = ['..'] * step + self._tail[len(path._tail):] return self._from_parsed_parts('', '', parts) - def is_relative_to(self, other, /, *_deprecated): + def is_relative_to(self, other): """Return True if the path is relative to another path or False. """ - if _deprecated: - msg = ("support for supplying more than one argument to " - "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): + if not isinstance(other, PurePath): other = self.with_segments(other) return other == self or other in self.parents diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 5fd1a41cbee17b..4fd2aac4a62139 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -311,19 +311,6 @@ def test_with_stem_empty(self): self.assertRaises(ValueError, P('a/b').with_stem, '') self.assertRaises(ValueError, P('a/b').with_stem, '.') - def test_relative_to_several_args(self): - P = self.cls - p = P('a/b') - with self.assertWarns(DeprecationWarning): - p.relative_to('a', 'b') - p.relative_to('a', 'b', walk_up=True) - - def test_is_relative_to_several_args(self): - P = self.cls - p = P('a/b') - with self.assertWarns(DeprecationWarning): - p.is_relative_to('a', 'b') - def test_is_reserved_deprecated(self): P = self.cls p = P('a/b') diff --git a/Misc/NEWS.d/next/Library/2024-05-08-18-59-19.gh-issue-78707._Lz1sw.rst b/Misc/NEWS.d/next/Library/2024-05-08-18-59-19.gh-issue-78707._Lz1sw.rst new file mode 100644 index 00000000000000..c73bab97b75838 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-18-59-19.gh-issue-78707._Lz1sw.rst @@ -0,0 +1,3 @@ +Drop support for passing additional positional arguments to +:meth:`pathlib.PurePath.relative_to` and +:meth:`~pathlib.PurePath.is_relative_to`. From 941eea0a27de6e5c02d3c80924235a7a07bd095a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 May 2024 18:20:12 +0200 Subject: [PATCH 046/903] gh-118771: Ensure names defined in optimizer.h start with Py/_Py (GH-118825) --- Include/cpython/optimizer.h | 41 +++---------------- Include/internal/pycore_optimizer.h | 29 +++++++++++++ ...-05-10-15-43-14.gh-issue-118771.5KVglT.rst | 3 ++ Python/optimizer.c | 8 ++-- 4 files changed, 42 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-05-10-15-43-14.gh-issue-118771.5KVglT.rst diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 5f218d75b346a0..f2093a1e5f6aa4 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -14,10 +14,10 @@ typedef struct _PyExecutorLinkListNode { /* Bloom filter with m = 256 * https://en.wikipedia.org/wiki/Bloom_filter */ -#define BLOOM_FILTER_WORDS 8 +#define _Py_BLOOM_FILTER_WORDS 8 -typedef struct _bloom_filter { - uint32_t bits[BLOOM_FILTER_WORDS]; +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; } _PyBloomFilter; typedef struct { @@ -31,11 +31,6 @@ typedef struct { PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). } _PyVMData; -#define UOP_FORMAT_TARGET 0 -#define UOP_FORMAT_EXIT 1 -#define UOP_FORMAT_JUMP 2 -#define UOP_FORMAT_UNUSED 3 - /* Depending on the format, * the 32 bits between the oparg and operand are: * UOP_FORMAT_TARGET: @@ -64,31 +59,7 @@ typedef struct { uint64_t operand; // A cache entry } _PyUOpInstruction; -static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) -{ - assert(inst->format == UOP_FORMAT_TARGET); - return inst->target; -} - -static inline uint16_t uop_get_exit_index(const _PyUOpInstruction *inst) -{ - assert(inst->format == UOP_FORMAT_EXIT); - return inst->exit_index; -} - -static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst) -{ - assert(inst->format == UOP_FORMAT_JUMP); - return inst->jump_target; -} - -static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) -{ - assert(inst->format != UOP_FORMAT_TARGET); - return inst->error_target; -} - -typedef struct _exit_data { +typedef struct { uint32_t target; _Py_BackoffCounter temperature; const struct _PyExecutorObject *executor; @@ -109,14 +80,14 @@ typedef struct _PyExecutorObject { typedef struct _PyOptimizerObject _PyOptimizerObject; /* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ -typedef int (*optimize_func)( +typedef int (*_Py_optimize_func)( _PyOptimizerObject* self, struct _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, int curr_stackentries); struct _PyOptimizerObject { PyObject_HEAD - optimize_func optimize; + _Py_optimize_func optimize; /* Data needed by the optimizer goes here, but is opaque to the VM */ }; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index c0a76e85350541..c1148422c495b6 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -35,6 +35,35 @@ struct _Py_UopsSymbol { PyObject *const_val; // Owned reference (!) }; +#define UOP_FORMAT_TARGET 0 +#define UOP_FORMAT_EXIT 1 +#define UOP_FORMAT_JUMP 2 +#define UOP_FORMAT_UNUSED 3 + +static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_TARGET); + return inst->target; +} + +static inline uint16_t uop_get_exit_index(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_EXIT); + return inst->exit_index; +} + +static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_JUMP); + return inst->jump_target; +} + +static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) +{ + assert(inst->format != UOP_FORMAT_TARGET); + return inst->error_target; +} + // Holds locals, stack, locals, stack ... co_consts (in that order) #define MAX_ABSTRACT_INTERP_SIZE 4096 diff --git a/Misc/NEWS.d/next/C API/2024-05-10-15-43-14.gh-issue-118771.5KVglT.rst b/Misc/NEWS.d/next/C API/2024-05-10-15-43-14.gh-issue-118771.5KVglT.rst new file mode 100644 index 00000000000000..2ed8089dfe8444 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-10-15-43-14.gh-issue-118771.5KVglT.rst @@ -0,0 +1,3 @@ +Several C declarations with names that didn't start with the ``Py`` or ``_Py`` +prefixes, which were added by mistake in 3.13 alpha and beta releases, were +moved to internal headers. diff --git a/Python/optimizer.c b/Python/optimizer.c index 8be2c0ffbd78e9..9ae99ccdaea2e7 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1237,7 +1237,7 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) inst->oparg = oparg; executor->vm_data.valid = true; executor->vm_data.linked = false; - for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { assert(executor->vm_data.bloom.bits[i] == 0); } #ifdef Py_DEBUG @@ -1505,7 +1505,7 @@ address_to_hash(void *ptr) { void _Py_BloomFilter_Init(_PyBloomFilter *bloom) { - for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { bloom->bits[i] = 0; } } @@ -1530,7 +1530,7 @@ _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *ptr) static bool bloom_filter_may_contain(_PyBloomFilter *bloom, _PyBloomFilter *hashes) { - for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { if ((bloom->bits[i] & hashes->bits[i]) != hashes->bits[i]) { return false; } @@ -1591,7 +1591,7 @@ void _Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_set) { executor->vm_data.valid = true; - for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { executor->vm_data.bloom.bits[i] = dependency_set->bits[i]; } link_executor(executor); From f5c6b9977a561fcf9c2a803fb08652fd39b13d3b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 May 2024 17:43:23 +0100 Subject: [PATCH 047/903] GH-118910: Less boilerplate in the tier 2 optimizer (#118913) --- Include/internal/pycore_optimizer.h | 14 +- Lib/test/test_generated_cases.py | 2 - Python/optimizer_analysis.c | 80 ++--- Python/optimizer_bytecodes.c | 234 ++++++-------- Python/optimizer_cases.c.h | 309 ++++++------------- Python/optimizer_symbols.c | 106 +++---- Tools/cases_generator/optimizer_generator.py | 3 - 7 files changed, 275 insertions(+), 473 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index c1148422c495b6..76123987ac99f5 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -93,7 +93,9 @@ typedef struct ty_arena { } ty_arena; struct _Py_UOpsContext { - PyObject_HEAD + char done; + char out_of_space; + bool contradiction; // The current "executing" frame. _Py_UOpsAbstractFrame *frame; _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; @@ -121,16 +123,16 @@ extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *con extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym); extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_null(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val); +extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); +extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ); +extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val); extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym); extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsSymbol *sym); -extern int _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); +extern void _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 7b9dd36f85454f..fb85222fdcce74 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -909,7 +909,6 @@ def test_overridden_abstract_args(self): case OP2: { _Py_UopsSymbol *out; out = sym_new_not_null(ctx); - if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; } @@ -934,7 +933,6 @@ def test_no_overridden_case(self): case OP: { _Py_UopsSymbol *out; out = sym_new_not_null(ctx); - if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 842b2e489239af..e5d3793bd4d204 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -297,20 +297,6 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, INST->oparg = ARG; \ INST->operand = OPERAND; -#define OUT_OF_SPACE_IF_NULL(EXPR) \ - do { \ - if ((EXPR) == NULL) { \ - goto out_of_space; \ - } \ - } while (0); - -#define _LOAD_ATTR_NOT_NULL \ - do { \ - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); \ - OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); \ - } while (0); - - /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null #define sym_is_const _Py_uop_sym_is_const @@ -324,10 +310,10 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_has_type _Py_uop_sym_has_type #define sym_get_type _Py_uop_sym_get_type #define sym_matches_type _Py_uop_sym_matches_type -#define sym_set_null _Py_uop_sym_set_null -#define sym_set_non_null _Py_uop_sym_set_non_null -#define sym_set_type _Py_uop_sym_set_type -#define sym_set_const _Py_uop_sym_set_const +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) #define sym_is_bottom _Py_uop_sym_is_bottom #define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new @@ -408,18 +394,20 @@ optimize_uops( _PyUOpInstruction *first_valid_check_stack = NULL; _PyUOpInstruction *corresponding_check_stack = NULL; - if (_Py_uop_abstractcontext_init(ctx) < 0) { - goto out_of_space; - } + _Py_uop_abstractcontext_init(ctx); _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); if (frame == NULL) { return -1; } ctx->curr_frame_depth++; ctx->frame = frame; + ctx->done = false; + ctx->out_of_space = false; + ctx->contradiction = false; _PyUOpInstruction *this_instr = NULL; - for (int i = 0; i < trace_len; i++) { + for (int i = 0; !ctx->done; i++) { + assert(i < trace_len); this_instr = &trace[i]; int oparg = this_instr->oparg; @@ -447,32 +435,22 @@ optimize_uops( ctx->frame->stack_pointer = stack_pointer; assert(STACK_LEVEL() >= 0); } - Py_UNREACHABLE(); - -out_of_space: - DPRINTF(3, "\n"); - DPRINTF(1, "Out of space in abstract interpreter\n"); - goto done; -error: - DPRINTF(3, "\n"); - DPRINTF(1, "Encountered error in abstract interpreter\n"); - if (opcode <= MAX_UOP_ID) { - OPT_ERROR_IN_OPCODE(opcode); + if (ctx->out_of_space) { + DPRINTF(3, "\n"); + DPRINTF(1, "Out of space in abstract interpreter\n"); + } + if (ctx->contradiction) { + // Attempted to push a "bottom" (contradiction) symbol onto the stack. + // This means that the abstract interpreter has hit unreachable code. + // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting + // bottom indicates type instability, so we are probably better off + // retrying later. + DPRINTF(3, "\n"); + DPRINTF(1, "Hit bottom in abstract interpreter\n"); + _Py_uop_abstractcontext_fini(ctx); + return 0; } - _Py_uop_abstractcontext_fini(ctx); - return -1; -hit_bottom: - // Attempted to push a "bottom" (contradition) symbol onto the stack. - // This means that the abstract interpreter has hit unreachable code. - // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting - // bottom indicates type instability, so we are probably better off - // retrying later. - DPRINTF(3, "\n"); - DPRINTF(1, "Hit bottom in abstract interpreter\n"); - _Py_uop_abstractcontext_fini(ctx); - return 0; -done: /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); @@ -485,6 +463,16 @@ optimize_uops( first_valid_check_stack->operand = max_space; } return trace_len; + +error: + DPRINTF(3, "\n"); + DPRINTF(1, "Encountered error in abstract interpreter\n"); + if (opcode <= MAX_UOP_ID) { + OPT_ERROR_IN_OPCODE(opcode); + } + _Py_uop_abstractcontext_fini(ctx); + return -1; + } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 928bc03382b8fb..e5c982befb2411 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -23,10 +23,10 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_matches_type _Py_uop_sym_matches_type #define sym_get_type _Py_uop_sym_get_type #define sym_has_type _Py_uop_sym_has_type -#define sym_set_null _Py_uop_sym_set_null -#define sym_set_non_null _Py_uop_sym_set_non_null -#define sym_set_type _Py_uop_sym_set_type -#define sym_set_const _Py_uop_sym_set_const +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) #define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop @@ -73,7 +73,7 @@ dummy_func(void) { value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { - goto out_of_space; + ctx->done = true; } } @@ -83,8 +83,7 @@ dummy_func(void) { op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - _Py_UopsSymbol *temp; - OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); + _Py_UopsSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } @@ -94,9 +93,6 @@ dummy_func(void) { op(_PUSH_NULL, (-- res)) { res = sym_new_null(ctx); - if (res == NULL) { - goto out_of_space; - }; } op(_GUARD_BOTH_INT, (left, right -- left, right)) { @@ -113,12 +109,8 @@ dummy_func(void) { REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); } } - if (!sym_set_type(left, &PyLong_Type)) { - goto hit_bottom; - } - if (!sym_set_type(right, &PyLong_Type)) { - goto hit_bottom; - } + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); } op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { @@ -135,12 +127,9 @@ dummy_func(void) { REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); } } - if (!sym_set_type(left, &PyFloat_Type)) { - goto hit_bottom; - } - if (!sym_set_type(right, &PyFloat_Type)) { - goto hit_bottom; - } + + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); } op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { @@ -148,12 +137,8 @@ dummy_func(void) { sym_matches_type(right, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - if (!sym_set_type(left, &PyUnicode_Type)) { - goto hit_bottom; - } - if (!sym_set_type(right, &PyUnicode_Type)) { - goto hit_bottom; - } + sym_set_type(left, &PyUnicode_Type); + sym_set_type(left, &PyUnicode_Type); } op(_BINARY_OP, (left, right -- res)) { @@ -165,14 +150,14 @@ dummy_func(void) { if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && ltype == &PyLong_Type && rtype == &PyLong_Type) { /* If both inputs are ints and the op is not division the result is an int */ - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } else { /* For any other op combining ints/floats the result is a float */ - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } } - OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + res = sym_new_unknown(ctx); } op(_BINARY_OP_ADD_INT, (left, right -- res)) { @@ -188,12 +173,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } } @@ -210,12 +194,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } } @@ -232,12 +215,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } } @@ -255,12 +237,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } } @@ -278,12 +259,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } } @@ -301,12 +281,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } } @@ -319,80 +298,50 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyUnicode_Type)); + res = sym_new_type(ctx, &PyUnicode_Type); } } op(_TO_BOOL, (value -- res)) { - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { res = sym_new_type(ctx, &PyBool_Type); - OUT_OF_SPACE_IF_NULL(res); } } op(_TO_BOOL_BOOL, (value -- res)) { - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { - if(!sym_set_type(value, &PyBool_Type)) { - goto hit_bottom; - } + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyBool_Type); res = value; } } op(_TO_BOOL_INT, (value -- res)) { - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { - if(!sym_set_type(value, &PyLong_Type)) { - goto hit_bottom; - } - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyLong_Type); + res = sym_new_type(ctx, &PyBool_Type); } } op(_TO_BOOL_LIST, (value -- res)) { - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { - if(!sym_set_type(value, &PyList_Type)) { - goto hit_bottom; - } - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyList_Type); + res = sym_new_type(ctx, &PyBool_Type); } } op(_TO_BOOL_NONE, (value -- res)) { - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { - if (!sym_set_const(value, Py_None)) { - goto hit_bottom; - } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); } } op(_TO_BOOL_STR, (value -- res)) { - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); - if(!sym_set_type(value, &PyUnicode_Type)) { - goto hit_bottom; - } + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + sym_set_type(value, &PyUnicode_Type); } } @@ -400,66 +349,66 @@ dummy_func(void) { (void)left; (void)right; if (oparg & 16) { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_not_null(ctx)); + res = _Py_uop_sym_new_not_null(ctx); } } op(_COMPARE_OP_INT, (left, right -- res)) { (void)left; (void)right; - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); } op(_COMPARE_OP_FLOAT, (left, right -- res)) { (void)left; (void)right; - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); } op(_COMPARE_OP_STR, (left, right -- res)) { (void)left; (void)right; - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); } op(_IS_OP, (left, right -- res)) { (void)left; (void)right; - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP, (left, right -- res)) { (void)left; (void)right; - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); } op(_LOAD_CONST, (-- value)) { PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, val)); + value = sym_new_const(ctx, val); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + value = sym_new_const(ctx, ptr); } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + value = sym_new_const(ctx, ptr); } op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); } op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); } op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { @@ -472,7 +421,8 @@ dummy_func(void) { } op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)index; (void)owner; } @@ -496,15 +446,15 @@ dummy_func(void) { op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { (void)owner; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); if (oparg & 1) { - OUT_OF_SPACE_IF_NULL(self_or_null = sym_new_unknown(ctx)); + self_or_null = sym_new_unknown(ctx); } } op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { (void)index; - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + null = sym_new_null(ctx); attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. @@ -515,72 +465,69 @@ dummy_func(void) { PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { this_instr[-1].opcode = _POP_TOP; - OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); + attr = sym_new_const(ctx, res); } } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); } } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)hint; (void)owner; } op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)index; (void)owner; } op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)descr; (void)owner; } op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; } op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; } op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { (void)callable; - OUT_OF_SPACE_IF_NULL(func = sym_new_not_null(ctx)); - OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); + func = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); } op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - if (!sym_set_type(callable, &PyFunction_Type)) { - goto hit_bottom; - } + sym_set_type(callable, &PyFunction_Type); (void)self_or_null; (void)func_version; } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - if (!sym_set_null(null)) { - goto hit_bottom; - } - if (!sym_set_type(callable, &PyMethod_Type)) { - goto hit_bottom; - } + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -602,7 +549,8 @@ dummy_func(void) { if (func == NULL) { DPRINTF(3, "\n"); DPRINTF(1, "Missing function\n"); - goto done; + ctx->done = true; + break; } co = (PyCodeObject *)func->func_code; DPRINTF(3, "code=%p ", co); @@ -625,8 +573,7 @@ dummy_func(void) { localsplus_start = args; n_locals_already_filled = argcount; } - OUT_OF_SPACE_IF_NULL(new_frame = - frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + new_frame = frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0); } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -635,7 +582,8 @@ dummy_func(void) { (void)self_or_null; (void)args; first_valid_check_stack = NULL; - goto done; + new_frame = NULL; + ctx->done = true; } op(_POP_FRAME, (retval -- res)) { @@ -656,7 +604,7 @@ dummy_func(void) { co = get_code(this_instr); if (co == NULL) { // might be impossible, but bailing is still safe - goto done; + ctx->done = true; } } @@ -665,7 +613,7 @@ dummy_func(void) { ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + res = sym_new_unknown(ctx); /* Stack space handling */ assert(corresponding_check_stack == NULL); @@ -678,17 +626,17 @@ dummy_func(void) { co = get_code(this_instr); if (co == NULL) { // might be impossible, but bailing is still safe - goto done; + ctx->done = true; } } op(_YIELD_VALUE, (unused -- res)) { - OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + res = sym_new_unknown(ctx); } op(_FOR_ITER_GEN_FRAME, ( -- )) { /* We are about to hit the end of the trace */ - goto done; + ctx->done = true; } op(_CHECK_STACK_SPACE, ( --)) { @@ -712,7 +660,8 @@ dummy_func(void) { co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway - goto done; + ctx->done = true; + break; } /* Stack space handling */ @@ -721,7 +670,8 @@ dummy_func(void) { curr_space += framesize; if (curr_space < 0 || curr_space > INT32_MAX) { // won't fit in signed 32-bit int - goto done; + ctx->done = true; + break; } max_space = curr_space > max_space ? curr_space : max_space; if (first_valid_check_stack == NULL) { @@ -738,7 +688,7 @@ dummy_func(void) { /* This has to be done manually */ (void)seq; for (int i = 0; i < oparg; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + values[i] = sym_new_unknown(ctx); } } @@ -747,12 +697,12 @@ dummy_func(void) { (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + values[i] = sym_new_unknown(ctx); } } op(_ITER_NEXT_RANGE, (iter -- iter, next)) { - OUT_OF_SPACE_IF_NULL(next = sym_new_type(ctx, &PyLong_Type)); + next = sym_new_type(ctx, &PyLong_Type); (void)iter; } @@ -805,11 +755,11 @@ dummy_func(void) { } op(_JUMP_TO_TOP, (--)) { - goto done; + ctx->done = true; } op(_EXIT_TRACE, (--)) { - goto done; + ctx->done = true; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 2a4efd73d794df..cd4431d55ad908 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -18,7 +18,7 @@ value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { - goto out_of_space; + ctx->done = true; } stack_pointer[0] = value; stack_pointer += 1; @@ -36,8 +36,7 @@ case _LOAD_FAST_AND_CLEAR: { _Py_UopsSymbol *value; value = GETLOCAL(oparg); - _Py_UopsSymbol *temp; - OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); + _Py_UopsSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -49,7 +48,7 @@ PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, val)); + value = sym_new_const(ctx, val); stack_pointer[0] = value; stack_pointer += 1; break; @@ -71,9 +70,6 @@ case _PUSH_NULL: { _Py_UopsSymbol *res; res = sym_new_null(ctx); - if (res == NULL) { - goto out_of_space; - }; stack_pointer[0] = res; stack_pointer += 1; break; @@ -82,7 +78,6 @@ case _END_SEND: { _Py_UopsSymbol *value; value = sym_new_not_null(ctx); - if (value == NULL) goto out_of_space; stack_pointer[-2] = value; stack_pointer += -1; break; @@ -91,7 +86,6 @@ case _UNARY_NEGATIVE: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; } @@ -99,7 +93,6 @@ case _UNARY_NOT: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; } @@ -108,12 +101,8 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { res = sym_new_type(ctx, &PyBool_Type); - OUT_OF_SPACE_IF_NULL(res); } stack_pointer[-1] = res; break; @@ -123,13 +112,8 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { - if(!sym_set_type(value, &PyBool_Type)) { - goto hit_bottom; - } + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyBool_Type); res = value; } stack_pointer[-1] = res; @@ -140,14 +124,9 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { - if(!sym_set_type(value, &PyLong_Type)) { - goto hit_bottom; - } - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyLong_Type); + res = sym_new_type(ctx, &PyBool_Type); } stack_pointer[-1] = res; break; @@ -157,14 +136,9 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { - if(!sym_set_type(value, &PyList_Type)) { - goto hit_bottom; - } - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyList_Type); + res = sym_new_type(ctx, &PyBool_Type); } stack_pointer[-1] = res; break; @@ -174,14 +148,9 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { - if (!sym_set_const(value, Py_None)) { - goto hit_bottom; - } - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); } stack_pointer[-1] = res; break; @@ -191,14 +160,9 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - if (optimize_to_bool(this_instr, ctx, value, &res)) { - OUT_OF_SPACE_IF_NULL(res); - } - else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); - if(!sym_set_type(value, &PyUnicode_Type)) { - goto hit_bottom; - } + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + sym_set_type(value, &PyUnicode_Type); } stack_pointer[-1] = res; break; @@ -207,7 +171,6 @@ case _REPLACE_WITH_TRUE: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; } @@ -215,7 +178,6 @@ case _UNARY_INVERT: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; } @@ -238,12 +200,8 @@ REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); } } - if (!sym_set_type(left, &PyLong_Type)) { - goto hit_bottom; - } - if (!sym_set_type(right, &PyLong_Type)) { - goto hit_bottom; - } + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); break; } @@ -273,12 +231,11 @@ } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -303,12 +260,11 @@ } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -333,12 +289,11 @@ } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -363,12 +318,8 @@ REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); } } - if (!sym_set_type(left, &PyFloat_Type)) { - goto hit_bottom; - } - if (!sym_set_type(right, &PyFloat_Type)) { - goto hit_bottom; - } + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); break; } @@ -399,12 +350,11 @@ } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -430,12 +380,11 @@ } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -461,12 +410,11 @@ } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -482,12 +430,8 @@ sym_matches_type(right, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - if (!sym_set_type(left, &PyUnicode_Type)) { - goto hit_bottom; - } - if (!sym_set_type(right, &PyUnicode_Type)) { - goto hit_bottom; - } + sym_set_type(left, &PyUnicode_Type); + sym_set_type(left, &PyUnicode_Type); break; } @@ -505,10 +449,9 @@ } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyUnicode_Type)); + res = sym_new_type(ctx, &PyUnicode_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -518,7 +461,6 @@ case _BINARY_SUBSCR: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -527,7 +469,6 @@ case _BINARY_SLICE: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; break; @@ -541,7 +482,6 @@ case _BINARY_SUBSCR_LIST_INT: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -550,7 +490,6 @@ case _BINARY_SUBSCR_STR_INT: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -559,7 +498,6 @@ case _BINARY_SUBSCR_TUPLE_INT: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -568,7 +506,6 @@ case _BINARY_SUBSCR_DICT: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -609,7 +546,6 @@ case _CALL_INTRINSIC_1: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; } @@ -617,7 +553,6 @@ case _CALL_INTRINSIC_2: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -642,7 +577,7 @@ co = get_code(this_instr); if (co == NULL) { // might be impossible, but bailing is still safe - goto done; + ctx->done = true; } stack_pointer[0] = res; stack_pointer += 1; @@ -656,7 +591,6 @@ case _GET_AITER: { _Py_UopsSymbol *iter; iter = sym_new_not_null(ctx); - if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; } @@ -664,7 +598,6 @@ case _GET_ANEXT: { _Py_UopsSymbol *awaitable; awaitable = sym_new_not_null(ctx); - if (awaitable == NULL) goto out_of_space; stack_pointer[0] = awaitable; stack_pointer += 1; break; @@ -673,7 +606,6 @@ case _GET_AWAITABLE: { _Py_UopsSymbol *iter; iter = sym_new_not_null(ctx); - if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; } @@ -686,7 +618,7 @@ case _YIELD_VALUE: { _Py_UopsSymbol *res; - OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + res = sym_new_unknown(ctx); stack_pointer[-1] = res; break; } @@ -699,7 +631,6 @@ case _LOAD_ASSERTION_ERROR: { _Py_UopsSymbol *value; value = sym_new_not_null(ctx); - if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; break; @@ -708,7 +639,6 @@ case _LOAD_BUILD_CLASS: { _Py_UopsSymbol *bc; bc = sym_new_not_null(ctx); - if (bc == NULL) goto out_of_space; stack_pointer[0] = bc; stack_pointer += 1; break; @@ -731,7 +661,7 @@ /* This has to be done manually */ (void)seq; for (int i = 0; i < oparg; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + values[i] = sym_new_unknown(ctx); } stack_pointer += -1 + oparg; break; @@ -741,9 +671,7 @@ _Py_UopsSymbol *val1; _Py_UopsSymbol *val0; val1 = sym_new_not_null(ctx); - if (val1 == NULL) goto out_of_space; val0 = sym_new_not_null(ctx); - if (val0 == NULL) goto out_of_space; stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -755,7 +683,6 @@ values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); - if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; break; @@ -766,7 +693,6 @@ values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); - if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; break; @@ -781,7 +707,7 @@ (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + values[i] = sym_new_unknown(ctx); } stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; @@ -809,7 +735,6 @@ case _LOAD_LOCALS: { _Py_UopsSymbol *locals; locals = sym_new_not_null(ctx); - if (locals == NULL) goto out_of_space; stack_pointer[0] = locals; stack_pointer += 1; break; @@ -818,7 +743,6 @@ case _LOAD_FROM_DICT_OR_GLOBALS: { _Py_UopsSymbol *v; v = sym_new_not_null(ctx); - if (v == NULL) goto out_of_space; stack_pointer[-1] = v; break; } @@ -829,9 +753,7 @@ _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; null = sym_new_null(ctx); - if (null == NULL) goto out_of_space; stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); @@ -850,9 +772,7 @@ _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; null = sym_new_null(ctx); - if (null == NULL) goto out_of_space; stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); @@ -863,9 +783,7 @@ _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; null = sym_new_null(ctx); - if (null == NULL) goto out_of_space; stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); @@ -887,7 +805,6 @@ case _LOAD_FROM_DICT_OR_DEREF: { _Py_UopsSymbol *value; value = sym_new_not_null(ctx); - if (value == NULL) goto out_of_space; stack_pointer[-1] = value; break; } @@ -895,7 +812,6 @@ case _LOAD_DEREF: { _Py_UopsSymbol *value; value = sym_new_not_null(ctx); - if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; break; @@ -913,7 +829,6 @@ case _BUILD_STRING: { _Py_UopsSymbol *str; str = sym_new_not_null(ctx); - if (str == NULL) goto out_of_space; stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; break; @@ -922,7 +837,6 @@ case _BUILD_TUPLE: { _Py_UopsSymbol *tup; tup = sym_new_not_null(ctx); - if (tup == NULL) goto out_of_space; stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; break; @@ -931,7 +845,6 @@ case _BUILD_LIST: { _Py_UopsSymbol *list; list = sym_new_not_null(ctx); - if (list == NULL) goto out_of_space; stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; break; @@ -952,7 +865,6 @@ case _BUILD_MAP: { _Py_UopsSymbol *map; map = sym_new_not_null(ctx); - if (map == NULL) goto out_of_space; stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; break; @@ -965,7 +877,6 @@ case _BUILD_CONST_KEY_MAP: { _Py_UopsSymbol *map; map = sym_new_not_null(ctx); - if (map == NULL) goto out_of_space; stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; break; @@ -991,7 +902,6 @@ case _LOAD_SUPER_ATTR_ATTR: { _Py_UopsSymbol *attr; attr = sym_new_not_null(ctx); - if (attr == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer += -2; break; @@ -1001,9 +911,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null; attr = sym_new_not_null(ctx); - if (attr == NULL) goto out_of_space; self_or_null = sym_new_not_null(ctx); - if (self_or_null == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; @@ -1016,9 +924,9 @@ _Py_UopsSymbol *self_or_null = NULL; owner = stack_pointer[-1]; (void)owner; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); if (oparg & 1) { - OUT_OF_SPACE_IF_NULL(self_or_null = sym_new_unknown(ctx)); + self_or_null = sym_new_unknown(ctx); } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; @@ -1040,7 +948,8 @@ _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; - _LOAD_ATTR_NOT_NULL + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)index; (void)owner; stack_pointer[-1] = attr; @@ -1077,7 +986,7 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; (void)index; - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + null = sym_new_null(ctx); attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. @@ -1088,12 +997,12 @@ PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { this_instr[-1].opcode = _POP_TOP; - OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); + attr = sym_new_const(ctx, res); } } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; @@ -1111,7 +1020,8 @@ _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t hint = (uint16_t)this_instr->operand; - _LOAD_ATTR_NOT_NULL + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)hint; (void)owner; stack_pointer[-1] = attr; @@ -1126,7 +1036,8 @@ _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; - _LOAD_ATTR_NOT_NULL + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)index; (void)owner; stack_pointer[-1] = attr; @@ -1145,7 +1056,8 @@ _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; - _LOAD_ATTR_NOT_NULL + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)descr; (void)owner; stack_pointer[-1] = attr; @@ -1183,10 +1095,10 @@ (void)left; (void)right; if (oparg & 16) { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); } else { - OUT_OF_SPACE_IF_NULL(res = _Py_uop_sym_new_not_null(ctx)); + res = _Py_uop_sym_new_not_null(ctx); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1201,7 +1113,7 @@ left = stack_pointer[-2]; (void)left; (void)right; - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -1215,7 +1127,7 @@ left = stack_pointer[-2]; (void)left; (void)right; - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -1229,7 +1141,7 @@ left = stack_pointer[-2]; (void)left; (void)right; - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -1243,7 +1155,7 @@ left = stack_pointer[-2]; (void)left; (void)right; - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -1257,7 +1169,7 @@ left = stack_pointer[-2]; (void)left; (void)right; - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -1266,7 +1178,6 @@ case _CONTAINS_OP_SET: { _Py_UopsSymbol *b; b = sym_new_not_null(ctx); - if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; break; @@ -1275,7 +1186,6 @@ case _CONTAINS_OP_DICT: { _Py_UopsSymbol *b; b = sym_new_not_null(ctx); - if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; break; @@ -1285,9 +1195,7 @@ _Py_UopsSymbol *rest; _Py_UopsSymbol *match; rest = sym_new_not_null(ctx); - if (rest == NULL) goto out_of_space; match = sym_new_not_null(ctx); - if (match == NULL) goto out_of_space; stack_pointer[-2] = rest; stack_pointer[-1] = match; break; @@ -1296,7 +1204,6 @@ case _CHECK_EXC_MATCH: { _Py_UopsSymbol *b; b = sym_new_not_null(ctx); - if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; } @@ -1308,7 +1215,6 @@ case _IS_NONE: { _Py_UopsSymbol *b; b = sym_new_not_null(ctx); - if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; } @@ -1316,7 +1222,6 @@ case _GET_LEN: { _Py_UopsSymbol *len_o; len_o = sym_new_not_null(ctx); - if (len_o == NULL) goto out_of_space; stack_pointer[0] = len_o; stack_pointer += 1; break; @@ -1325,7 +1230,6 @@ case _MATCH_CLASS: { _Py_UopsSymbol *attrs; attrs = sym_new_not_null(ctx); - if (attrs == NULL) goto out_of_space; stack_pointer[-3] = attrs; stack_pointer += -2; break; @@ -1334,7 +1238,6 @@ case _MATCH_MAPPING: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; break; @@ -1343,7 +1246,6 @@ case _MATCH_SEQUENCE: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; break; @@ -1352,7 +1254,6 @@ case _MATCH_KEYS: { _Py_UopsSymbol *values_or_none; values_or_none = sym_new_not_null(ctx); - if (values_or_none == NULL) goto out_of_space; stack_pointer[0] = values_or_none; stack_pointer += 1; break; @@ -1361,7 +1262,6 @@ case _GET_ITER: { _Py_UopsSymbol *iter; iter = sym_new_not_null(ctx); - if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; } @@ -1369,7 +1269,6 @@ case _GET_YIELD_FROM_ITER: { _Py_UopsSymbol *iter; iter = sym_new_not_null(ctx); - if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; } @@ -1379,7 +1278,6 @@ case _FOR_ITER_TIER_TWO: { _Py_UopsSymbol *next; next = sym_new_not_null(ctx); - if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; break; @@ -1400,7 +1298,6 @@ case _ITER_NEXT_LIST: { _Py_UopsSymbol *next; next = sym_new_not_null(ctx); - if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; break; @@ -1419,7 +1316,6 @@ case _ITER_NEXT_TUPLE: { _Py_UopsSymbol *next; next = sym_new_not_null(ctx); - if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; break; @@ -1439,7 +1335,7 @@ _Py_UopsSymbol *iter; _Py_UopsSymbol *next; iter = stack_pointer[-1]; - OUT_OF_SPACE_IF_NULL(next = sym_new_type(ctx, &PyLong_Type)); + next = sym_new_type(ctx, &PyLong_Type); (void)iter; stack_pointer[0] = next; stack_pointer += 1; @@ -1448,7 +1344,7 @@ case _FOR_ITER_GEN_FRAME: { /* We are about to hit the end of the trace */ - goto done; + ctx->done = true; break; } @@ -1459,7 +1355,6 @@ case _WITH_EXCEPT_START: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; break; @@ -1469,9 +1364,7 @@ _Py_UopsSymbol *prev_exc; _Py_UopsSymbol *new_exc; prev_exc = sym_new_not_null(ctx); - if (prev_exc == NULL) goto out_of_space; new_exc = sym_new_not_null(ctx); - if (new_exc == NULL) goto out_of_space; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -1493,7 +1386,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -1508,7 +1401,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -1519,7 +1412,6 @@ case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { _Py_UopsSymbol *attr; attr = sym_new_not_null(ctx); - if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; } @@ -1527,7 +1419,6 @@ case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { _Py_UopsSymbol *attr; attr = sym_new_not_null(ctx); - if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; } @@ -1543,7 +1434,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -1572,7 +1463,8 @@ (void)self_or_null; (void)args; first_valid_check_stack = NULL; - goto done; + new_frame = NULL; + ctx->done = true; stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1 - oparg; break; @@ -1590,9 +1482,7 @@ _Py_UopsSymbol *method; _Py_UopsSymbol *self; method = sym_new_not_null(ctx); - if (method == NULL) goto out_of_space; self = sym_new_not_null(ctx); - if (self == NULL) goto out_of_space; stack_pointer[-2 - oparg] = method; stack_pointer[-1 - oparg] = self; break; @@ -1605,7 +1495,6 @@ case _CALL_NON_PY_GENERAL: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1616,12 +1505,8 @@ _Py_UopsSymbol *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - if (!sym_set_null(null)) { - goto hit_bottom; - } - if (!sym_set_type(callable, &PyMethod_Type)) { - goto hit_bottom; - } + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); break; } @@ -1631,8 +1516,8 @@ _Py_UopsSymbol *self; callable = stack_pointer[-2 - oparg]; (void)callable; - OUT_OF_SPACE_IF_NULL(func = sym_new_not_null(ctx)); - OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); + func = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = self; break; @@ -1653,9 +1538,7 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand; - if (!sym_set_type(callable, &PyFunction_Type)) { - goto hit_bottom; - } + sym_set_type(callable, &PyFunction_Type); (void)self_or_null; (void)func_version; break; @@ -1691,7 +1574,8 @@ if (func == NULL) { DPRINTF(3, "\n"); DPRINTF(1, "Missing function\n"); - goto done; + ctx->done = true; + break; } co = (PyCodeObject *)func->func_code; DPRINTF(3, "code=%p ", co); @@ -1712,8 +1596,7 @@ localsplus_start = args; n_locals_already_filled = argcount; } - OUT_OF_SPACE_IF_NULL(new_frame = - frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + new_frame = frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0); stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1 - oparg; break; @@ -1730,7 +1613,8 @@ co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway - goto done; + ctx->done = true; + break; } /* Stack space handling */ int framesize = co->co_framesize; @@ -1738,7 +1622,8 @@ curr_space += framesize; if (curr_space < 0 || curr_space > INT32_MAX) { // won't fit in signed 32-bit int - goto done; + ctx->done = true; + break; } max_space = curr_space > max_space ? curr_space : max_space; if (first_valid_check_stack == NULL) { @@ -1755,7 +1640,6 @@ case _CALL_TYPE_1: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; break; @@ -1764,7 +1648,6 @@ case _CALL_STR_1: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; break; @@ -1773,7 +1656,6 @@ case _CALL_TUPLE_1: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; break; @@ -1789,7 +1671,6 @@ case _CALL_BUILTIN_CLASS: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1798,7 +1679,6 @@ case _CALL_BUILTIN_O: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1807,7 +1687,6 @@ case _CALL_BUILTIN_FAST: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1816,7 +1695,6 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1825,7 +1703,6 @@ case _CALL_LEN: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1834,7 +1711,6 @@ case _CALL_ISINSTANCE: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1843,7 +1719,6 @@ case _CALL_METHOD_DESCRIPTOR_O: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1852,7 +1727,6 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1861,7 +1735,6 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1870,7 +1743,6 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -1887,7 +1759,6 @@ case _MAKE_FUNCTION: { _Py_UopsSymbol *func; func = sym_new_not_null(ctx); - if (func == NULL) goto out_of_space; stack_pointer[-1] = func; break; } @@ -1895,7 +1766,6 @@ case _SET_FUNCTION_ATTRIBUTE: { _Py_UopsSymbol *func; func = sym_new_not_null(ctx); - if (func == NULL) goto out_of_space; stack_pointer[-2] = func; stack_pointer += -1; break; @@ -1906,7 +1776,7 @@ ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + res = sym_new_unknown(ctx); /* Stack space handling */ assert(corresponding_check_stack == NULL); assert(co != NULL); @@ -1917,7 +1787,7 @@ co = get_code(this_instr); if (co == NULL) { // might be impossible, but bailing is still safe - goto done; + ctx->done = true; } stack_pointer[0] = res; stack_pointer += 1; @@ -1927,7 +1797,6 @@ case _BUILD_SLICE: { _Py_UopsSymbol *slice; slice = sym_new_not_null(ctx); - if (slice == NULL) goto out_of_space; stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); break; @@ -1936,7 +1805,6 @@ case _CONVERT_VALUE: { _Py_UopsSymbol *result; result = sym_new_not_null(ctx); - if (result == NULL) goto out_of_space; stack_pointer[-1] = result; break; } @@ -1944,7 +1812,6 @@ case _FORMAT_SIMPLE: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; } @@ -1952,7 +1819,6 @@ case _FORMAT_WITH_SPEC: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -1983,14 +1849,14 @@ if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && ltype == &PyLong_Type && rtype == &PyLong_Type) { /* If both inputs are ints and the op is not division the result is an int */ - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } else { /* For any other op combining ints/floats the result is a float */ - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } } - OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + res = sym_new_unknown(ctx); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -2077,7 +1943,7 @@ } case _JUMP_TO_TOP: { - goto done; + ctx->done = true; break; } @@ -2099,7 +1965,7 @@ } case _EXIT_TRACE: { - goto done; + ctx->done = true; break; } @@ -2110,7 +1976,7 @@ case _LOAD_CONST_INLINE: { _Py_UopsSymbol *value; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + value = sym_new_const(ctx, ptr); stack_pointer[0] = value; stack_pointer += 1; break; @@ -2119,7 +1985,7 @@ case _LOAD_CONST_INLINE_BORROW: { _Py_UopsSymbol *value; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + value = sym_new_const(ctx, ptr); stack_pointer[0] = value; stack_pointer += 1; break; @@ -2128,7 +1994,6 @@ case _POP_TOP_LOAD_CONST_INLINE_BORROW: { _Py_UopsSymbol *value; value = sym_new_not_null(ctx); - if (value == NULL) goto out_of_space; stack_pointer[-1] = value; break; } @@ -2137,8 +2002,8 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *null; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; @@ -2149,8 +2014,8 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *null; PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 4aeb04fe0405d2..e546eef306eeca 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -32,6 +32,7 @@ // Flags for below. #define IS_NULL 1 << 0 #define NOT_NULL 1 << 1 +#define NO_SPACE 1 << 2 #ifdef Py_DEBUG static inline int get_lltrace(void) { @@ -48,6 +49,20 @@ static inline int get_lltrace(void) { #define DPRINTF(level, ...) #endif +static _Py_UopsSymbol NO_SPACE_SYMBOL = { + .flags = IS_NULL | NOT_NULL | NO_SPACE, + .typ = NULL, + .const_val = NULL +}; + +_Py_UopsSymbol * +out_of_space(_Py_UOpsContext *ctx) +{ + ctx->done = true; + ctx->out_of_space = true; + return &NO_SPACE_SYMBOL; +} + static _Py_UopsSymbol * sym_new(_Py_UOpsContext *ctx) { @@ -72,11 +87,13 @@ sym_set_flag(_Py_UopsSymbol *sym, int flag) } static inline void -sym_set_bottom(_Py_UopsSymbol *sym) +sym_set_bottom(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) { sym_set_flag(sym, IS_NULL | NOT_NULL); sym->typ = NULL; Py_CLEAR(sym->const_val); + ctx->done = true; + ctx->contradiction = true; } bool @@ -115,45 +132,41 @@ _Py_uop_sym_get_const(_Py_UopsSymbol *sym) return sym->const_val; } -bool -_Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +void +_Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); if (sym->flags & IS_NULL) { - sym_set_bottom(sym); - return false; + sym_set_bottom(ctx, sym); + return; } if (sym->typ != NULL) { if (sym->typ != typ) { - sym_set_bottom(sym); - return false; + sym_set_bottom(ctx, sym); + return; } } else { sym_set_flag(sym, NOT_NULL); sym->typ = typ; } - return true; } -bool -_Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val) +void +_Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val) { assert(const_val != NULL); if (sym->flags & IS_NULL) { - sym_set_bottom(sym); - return false; + sym_set_bottom(ctx, sym); } PyTypeObject *typ = Py_TYPE(const_val); if (sym->typ != NULL && sym->typ != typ) { - sym_set_bottom(sym); - return false; + sym_set_bottom(ctx, sym); } if (sym->const_val != NULL) { if (sym->const_val != const_val) { // TODO: What if they're equal? - sym_set_bottom(sym); - return false; + sym_set_bottom(ctx, sym); } } else { @@ -161,29 +174,24 @@ _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val) sym->typ = typ; sym->const_val = Py_NewRef(const_val); } - return true; } -bool -_Py_uop_sym_set_null(_Py_UopsSymbol *sym) +void +_Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) { if (_Py_uop_sym_is_not_null(sym)) { - sym_set_bottom(sym); - return false; + sym_set_bottom(ctx, sym); } sym_set_flag(sym, IS_NULL); - return true; } -bool -_Py_uop_sym_set_non_null(_Py_UopsSymbol *sym) +void +_Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) { if (_Py_uop_sym_is_null(sym)) { - sym_set_bottom(sym); - return false; + sym_set_bottom(ctx, sym); } sym_set_flag(sym, NOT_NULL); - return true; } @@ -198,7 +206,7 @@ _Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) { _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); if (res == NULL) { - return NULL; + return out_of_space(ctx); } sym_set_flag(res, NOT_NULL); return res; @@ -209,9 +217,9 @@ _Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) { _Py_UopsSymbol *res = sym_new(ctx); if (res == NULL) { - return NULL; + return out_of_space(ctx); } - _Py_uop_sym_set_type(res, typ); + _Py_uop_sym_set_type(ctx, res, typ); return res; } @@ -222,9 +230,9 @@ _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) assert(const_val != NULL); _Py_UopsSymbol *res = sym_new(ctx); if (res == NULL) { - return NULL; + return out_of_space(ctx); } - _Py_uop_sym_set_const(res, const_val); + _Py_uop_sym_set_const(ctx, res, const_val); return res; } @@ -233,9 +241,9 @@ _Py_uop_sym_new_null(_Py_UOpsContext *ctx) { _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); if (null_sym == NULL) { - return NULL; + return out_of_space(ctx); } - _Py_uop_sym_set_null(null_sym); + _Py_uop_sym_set_null(ctx, null_sym); return null_sym; } @@ -318,6 +326,8 @@ _Py_uop_frame_new( frame->stack_pointer = frame->stack + curr_stackentries; ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize); if (ctx->n_consumed >= ctx->limit) { + ctx->done = true; + ctx->out_of_space = true; return NULL; } @@ -325,9 +335,6 @@ _Py_uop_frame_new( // Initialize with the initial state of all local variables for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) { _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); - if (local == NULL) { - return NULL; - } frame->locals[i] = local; } @@ -335,9 +342,6 @@ _Py_uop_frame_new( // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); - if (stackvar == NULL) { - return NULL; - } frame->stack[i] = stackvar; } @@ -357,7 +361,7 @@ _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx) } } -int +void _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) { ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; @@ -374,8 +378,6 @@ _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) // Frame setup ctx->curr_frame_depth = 0; - - return 0; } int @@ -404,8 +406,8 @@ static _Py_UopsSymbol * make_bottom(_Py_UOpsContext *ctx) { _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); - _Py_uop_sym_set_null(sym); - _Py_uop_sym_set_non_null(sym); + _Py_uop_sym_set_null(ctx, sym); + _Py_uop_sym_set_non_null(ctx, sym); return sym; } @@ -452,10 +454,10 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "int is a constant"); TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "int as constant is not NULL"); - _Py_uop_sym_set_type(sym, &PyLong_Type); // Should be a no-op + _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); - _Py_uop_sym_set_type(sym, &PyFloat_Type); // Should make it bottom + _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); val_42 = PyLong_FromLong(42); @@ -470,7 +472,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } - _Py_uop_sym_set_const(sym, val_42); + _Py_uop_sym_set_const(ctx, sym, val_42); TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 1, "bool(42) is not True"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); @@ -480,19 +482,19 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_get_const(sym) != NULL, "42 as constant is NULL"); TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "42 as constant isn't 42"); - _Py_uop_sym_set_type(sym, &PyLong_Type); // Should be a no-op + _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "(42 and 42) as constant isn't 42"); - _Py_uop_sym_set_type(sym, &PyFloat_Type); // Should make it bottom + _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); if (sym == NULL) { goto fail; } - _Py_uop_sym_set_const(sym, val_42); - _Py_uop_sym_set_const(sym, val_43); // Should make it bottom + _Py_uop_sym_set_const(ctx, sym, val_42); + _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 1c6b708e82321a..d5592672a55514 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -84,14 +84,11 @@ def emit_default(out: CWriter, uop: Uop) -> None: if var.is_array(): out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") - out.emit(f"if ({var.name}[_i] == NULL) goto out_of_space;\n") out.emit("}\n") elif var.name == "null": out.emit(f"{var.name} = sym_new_null(ctx);\n") - out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") else: out.emit(f"{var.name} = sym_new_not_null(ctx);\n") - out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") def write_uop( From ec9d12be9648ee60a2eb02d67069d74f8b314df9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 10 May 2024 09:55:49 -0700 Subject: [PATCH 048/903] Rename typing._collect_parameters (#118900) Unfortunately, released versions of typing_extensions monkeypatch this function without the extra parameter, which makes it so things break badly if current main is used with typing_extensions. Fortunately, the monkeypatching is not needed on Python 3.13, because CPython now implements PEP 696. By renaming the function, we prevent the monkeypatch from breaking typing.py internals. We keep the old name (raising a DeprecationWarning) to help other external users who call it. --- Lib/test/test_typing.py | 13 ++++++++++++- Lib/typing.py | 24 +++++++++++++++++------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 81fea41e9b9823..f10b0aea3cd7b9 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -45,7 +45,7 @@ import weakref import types -from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings +from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper from test.typinganndata import ann_module695, mod_generics_cache, _typed_dict_helper @@ -6325,6 +6325,8 @@ def test_or(self): self.assertEqual(X | "x", Union[X, "x"]) self.assertEqual("x" | X, Union["x", X]) + +class InternalsTests(BaseTestCase): def test_deprecation_for_no_type_params_passed_to__evaluate(self): with self.assertWarnsRegex( DeprecationWarning, @@ -6350,6 +6352,15 @@ def test_deprecation_for_no_type_params_passed_to__evaluate(self): self.assertEqual(cm.filename, __file__) + def test_collect_parameters(self): + typing = import_helper.import_fresh_module("typing") + with self.assertWarnsRegex( + DeprecationWarning, + "The private _collect_parameters function is deprecated" + ) as cm: + typing._collect_parameters + self.assertEqual(cm.filename, __file__) + @lru_cache() def cached_func(x, y): diff --git a/Lib/typing.py b/Lib/typing.py index e75a627d226e50..434574559e04fc 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -256,15 +256,15 @@ def _type_repr(obj): return repr(obj) -def _collect_parameters(args, *, enforce_default_ordering: bool = True): - """Collect all type variables and parameter specifications in args +def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): + """Collect all type parameters in args in order of first appearance (lexicographic order). For example:: >>> P = ParamSpec('P') >>> T = TypeVar('T') - >>> _collect_parameters((T, Callable[P, T])) + >>> _collect_type_parameters((T, Callable[P, T])) (~T, ~P) """ # required type parameter cannot appear after parameter with default @@ -280,7 +280,7 @@ def _collect_parameters(args, *, enforce_default_ordering: bool = True): # `t` might be a tuple, when `ParamSpec` is substituted with # `[T, int]`, or `[int, *Ts]`, etc. for x in t: - for collected in _collect_parameters([x]): + for collected in _collect_type_parameters([x]): if collected not in parameters: parameters.append(collected) elif hasattr(t, '__typing_subst__'): @@ -320,7 +320,7 @@ def _check_generic_specialization(cls, arguments): if actual_len < expected_len: # If the parameter at index `actual_len` in the parameters list # has a default, then all parameters after it must also have - # one, because we validated as much in _collect_parameters(). + # one, because we validated as much in _collect_type_parameters(). # That means that no error needs to be raised here, despite # the number of arguments being passed not matching the number # of parameters: all parameters that aren't explicitly @@ -1255,7 +1255,7 @@ def _generic_init_subclass(cls, *args, **kwargs): if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _collect_parameters(cls.__orig_bases__) + tvars = _collect_type_parameters(cls.__orig_bases__) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. @@ -1417,7 +1417,7 @@ def __init__(self, origin, args, *, inst=True, name=None): self.__args__ = tuple(... if a is _TypingEllipsis else a for a in args) enforce_default_ordering = origin in (Generic, Protocol) - self.__parameters__ = _collect_parameters( + self.__parameters__ = _collect_type_parameters( args, enforce_default_ordering=enforce_default_ordering, ) @@ -3770,6 +3770,16 @@ def __getattr__(attr): elif attr in {"ContextManager", "AsyncContextManager"}: import contextlib obj = _alias(getattr(contextlib, f"Abstract{attr}"), 2, name=attr, defaults=(bool | None,)) + elif attr == "_collect_parameters": + import warnings + + depr_message = ( + "The private _collect_parameters function is deprecated and will be" + " removed in a future version of Python. Any use of private functions" + " is discouraged and may break in the future." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=2) + obj = _collect_type_parameters else: raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") globals()[attr] = obj From aa36f83c1670f1e41fa9432a20e5c4a88ee9012c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 May 2024 21:08:24 +0200 Subject: [PATCH 049/903] gh-118702: Implement vectorcall for BaseException (#118703) * BaseException_vectorcall() now creates a tuple from 'args' array. * Creation an exception using BaseException_vectorcall() is now a single function call, rather than having to call BaseException_new() and then BaseException_init(). Calling BaseException_init() is inefficient since it overrides the 'args' attribute. * _PyErr_SetKeyError() now uses PyObject_CallOneArg() to create the KeyError instance to use BaseException_vectorcall(). --- Lib/test/test_exceptions.py | 23 +++++++++++++++++++++ Objects/exceptions.c | 40 +++++++++++++++++++++++++++++++++++++ Python/errors.c | 9 +++++---- 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 3138f50076f1df..9460d1f1c864b9 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1817,6 +1817,29 @@ def test_memory_error_in_subinterp(self): rc, _, err = script_helper.assert_python_ok("-c", code) self.assertIn(b'MemoryError', err) + def test_keyerror_context(self): + # Make sure that _PyErr_SetKeyError() chains exceptions + try: + err1 = None + err2 = None + try: + d = {} + try: + raise ValueError("bug") + except Exception as exc: + err1 = exc + d[1] + except Exception as exc: + err2 = exc + + self.assertIsInstance(err1, ValueError) + self.assertIsInstance(err2, KeyError) + self.assertEqual(err2.__context__, err1) + finally: + # Break any potential reference cycle + exc1 = None + exc2 = None + class NameErrorTests(unittest.TestCase): def test_name_error_has_name(self): diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 63c461d34fb4ff..f9cd577c1c16be 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -78,6 +78,40 @@ BaseException_init(PyBaseExceptionObject *self, PyObject *args, PyObject *kwds) return 0; } + +static PyObject * +BaseException_vectorcall(PyObject *type_obj, PyObject * const*args, + size_t nargsf, PyObject *kwnames) +{ + PyTypeObject *type = _PyType_CAST(type_obj); + if (!_PyArg_NoKwnames(type->tp_name, kwnames)) { + return NULL; + } + + PyBaseExceptionObject *self; + self = (PyBaseExceptionObject *)type->tp_alloc(type, 0); + if (!self) { + return NULL; + } + + // The dict is created on the fly in PyObject_GenericSetAttr() + self->dict = NULL; + self->notes = NULL; + self->traceback = NULL; + self->cause = NULL; + self->context = NULL; + self->suppress_context = 0; + + self->args = _PyTuple_FromArray(args, PyVectorcall_NARGS(nargsf)); + if (!self->args) { + Py_DECREF(self); + return NULL; + } + + return (PyObject *)self; +} + + static int BaseException_clear(PyBaseExceptionObject *self) { @@ -486,6 +520,7 @@ static PyTypeObject _PyExc_BaseException = { (initproc)BaseException_init, /* tp_init */ 0, /* tp_alloc */ BaseException_new, /* tp_new */ + .tp_vectorcall = BaseException_vectorcall, }; /* the CPython API expects exceptions to be (PyObject *) - both a hold-over from the previous implementation and also allowing Python objects to be used @@ -3675,6 +3710,11 @@ _PyExc_InitTypes(PyInterpreterState *interp) if (_PyStaticType_InitBuiltin(interp, exc) < 0) { return -1; } + if (exc->tp_new == BaseException_new + && exc->tp_init == (initproc)BaseException_init) + { + exc->tp_vectorcall = BaseException_vectorcall; + } } return 0; } diff --git a/Python/errors.c b/Python/errors.c index 433253b8f9aada..ad6b7dbef075cc 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -257,13 +257,14 @@ void _PyErr_SetKeyError(PyObject *arg) { PyThreadState *tstate = _PyThreadState_GET(); - PyObject *tup = PyTuple_Pack(1, arg); - if (!tup) { + PyObject *exc = PyObject_CallOneArg(PyExc_KeyError, arg); + if (!exc) { /* caller will expect error to be set anyway */ return; } - _PyErr_SetObject(tstate, PyExc_KeyError, tup); - Py_DECREF(tup); + + _PyErr_SetObject(tstate, (PyObject*)Py_TYPE(exc), exc); + Py_DECREF(exc); } void From b309c8ebff011f27012367b046ff92eecbdd68a5 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 10 May 2024 16:29:29 -0400 Subject: [PATCH 050/903] gh-118846: Fix free-threading test failures when run sequentially (#118864) The free-threaded build currently immortalizes some objects once the first thread is started. This can lead to test failures depending on the order in which tests are run. This PR addresses those failures by suppressing immortalization or skipping the affected tests. --- Lib/test/seq_tests.py | 1 + Lib/test/test_capi/test_misc.py | 5 ++++- Lib/test/test_descr.py | 1 + Lib/test/test_gc.py | 20 ++++++++++++++++++-- Lib/test/test_inspect/test_inspect.py | 4 +++- Lib/test/test_module/__init__.py | 3 +++ Lib/test/test_trace.py | 4 +++- Lib/test/test_zoneinfo/test_zoneinfo.py | 3 ++- 8 files changed, 35 insertions(+), 6 deletions(-) diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index a41970d8f3f55a..719c9434a16820 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -426,6 +426,7 @@ def test_pickle(self): self.assertEqual(lst2, lst) self.assertNotEqual(id(lst2), id(lst)) + @support.suppress_immortalization() def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.type2test) support.check_free_after_iterating(self, reversed, self.type2test) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 020e8493e57c0c..ed42d7b64302f9 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -26,7 +26,8 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import requires_limited_api -from test.support import requires_gil_enabled, expected_failure_if_gil_disabled +from test.support import suppress_immortalization +from test.support import expected_failure_if_gil_disabled from test.support import Py_GIL_DISABLED from test.support.script_helper import assert_python_failure, assert_python_ok, run_python_until_end try: @@ -481,6 +482,7 @@ def test_heap_ctype_doc_and_text_signature(self): def test_null_type_doc(self): self.assertEqual(_testcapi.NullTpDocType.__doc__, None) + @suppress_immortalization() def test_subclass_of_heap_gc_ctype_with_tpdealloc_decrefs_once(self): class HeapGcCTypeSubclass(_testcapi.HeapGcCType): def __init__(self): @@ -498,6 +500,7 @@ def __init__(self): del subclass_instance self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapGcCTypeSubclass)) + @suppress_immortalization() def test_subclass_of_heap_gc_ctype_with_del_modifying_dunder_class_only_decrefs_once(self): class A(_testcapi.HeapGcCType): def __init__(self): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 18144c8cbb2f0a..c3f292467a6738 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5014,6 +5014,7 @@ def __new__(cls): cls.lst = [2**i for i in range(10000)] X.descr + @support.suppress_immortalization() def test_remove_subclass(self): # bpo-46417: when the last subclass of a type is deleted, # remove_subclass() clears the internal dictionary of subclasses: diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 52681dc18cfb86..906f9884d6792f 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -3,7 +3,8 @@ from test import support from test.support import (verbose, refcount_test, cpython_only, requires_subprocess, - requires_gil_enabled) + requires_gil_enabled, suppress_immortalization, + Py_GIL_DISABLED) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink from test.support.script_helper import assert_python_ok, make_script @@ -109,6 +110,7 @@ def test_tuple(self): del l self.assertEqual(gc.collect(), 2) + @suppress_immortalization() def test_class(self): class A: pass @@ -117,6 +119,7 @@ class A: del A self.assertNotEqual(gc.collect(), 0) + @suppress_immortalization() def test_newstyleclass(self): class A(object): pass @@ -133,6 +136,7 @@ class A: del a self.assertNotEqual(gc.collect(), 0) + @suppress_immortalization() def test_newinstance(self): class A(object): pass @@ -219,6 +223,7 @@ class B(object): self.fail("didn't find obj in garbage (finalizer)") gc.garbage.remove(obj) + @suppress_immortalization() def test_function(self): # Tricky: f -> d -> f, code should call d.clear() after the exec to # break the cycle. @@ -561,6 +566,7 @@ def test_get_referents(self): self.assertEqual(gc.get_referents(1, 'a', 4j), []) + @suppress_immortalization() def test_is_tracked(self): # Atomic built-in types are not tracked, user-defined objects and # mutable containers are. @@ -598,7 +604,9 @@ class UserFloatSlots(float): class UserIntSlots(int): __slots__ = () - self.assertTrue(gc.is_tracked(gc)) + if not Py_GIL_DISABLED: + # gh-117783: modules may be immortalized in free-threaded build + self.assertTrue(gc.is_tracked(gc)) self.assertTrue(gc.is_tracked(UserClass)) self.assertTrue(gc.is_tracked(UserClass())) self.assertTrue(gc.is_tracked(UserInt())) @@ -1347,6 +1355,10 @@ def callback(ignored): junk = [] i = 0 detector = GC_Detector() + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() while not detector.gc_happened: i += 1 if i > 10000: @@ -1415,6 +1427,10 @@ def __del__(self): detector = GC_Detector() junk = [] i = 0 + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() while not detector.gc_happened: i += 1 if i > 10000: diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 82e466e978624f..8bd13033490b81 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -34,7 +34,7 @@ except ImportError: ThreadPoolExecutor = None -from test.support import cpython_only, import_helper +from test.support import cpython_only, import_helper, suppress_immortalization from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support.import_helper import DirsOnSysPath, ready_to_import from test.support.os_helper import TESTFN, temp_cwd @@ -768,6 +768,7 @@ def test_getfile_builtin_function_or_method(self): inspect.getfile(list.append) self.assertIn('expected, got', str(e_append.exception)) + @suppress_immortalization() def test_getfile_class_without_module(self): class CM(type): @property @@ -2430,6 +2431,7 @@ def __getattribute__(self, attr): self.assertFalse(test.called) + @suppress_immortalization() def test_cache_does_not_cause_classes_to_persist(self): # regression test for gh-118013: # check that the internal _shadowed_dict cache does not cause diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index 98d1cbe824df12..952ba43f72504d 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -4,6 +4,7 @@ import weakref from test.support import gc_collect from test.support import import_helper +from test.support import suppress_immortalization from test.support.script_helper import assert_python_ok import sys @@ -103,6 +104,7 @@ def f(): gc_collect() self.assertEqual(f().__dict__["bar"], 4) + @suppress_immortalization() def test_clear_dict_in_ref_cycle(self): destroyed = [] m = ModuleType("foo") @@ -118,6 +120,7 @@ def __del__(self): gc_collect() self.assertEqual(destroyed, [1]) + @suppress_immortalization() def test_weakref(self): m = ModuleType("foo") wr = weakref.ref(m) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 93966ee31d0a01..7ff3fe4091dfa4 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,7 +1,7 @@ import os from pickle import dump import sys -from test.support import captured_stdout, requires_resource +from test.support import captured_stdout, requires_resource, requires_gil_enabled from test.support.os_helper import (TESTFN, rmtree, unlink) from test.support.script_helper import assert_python_ok, assert_python_failure import textwrap @@ -301,6 +301,7 @@ def test_loop_caller_importing(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') + @requires_gil_enabled("gh-117783: immortalization of types affects traced method names") def test_inst_method_calling(self): obj = TracedClass(20) self.tracer.runfunc(obj.inst_method_calling, 1) @@ -334,6 +335,7 @@ def setUp(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') + @requires_gil_enabled("gh-117783: immortalization of types affects traced method names") def test_loop_caller_importing(self): self.tracer.runfunc(traced_func_importing_caller, 1) diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 8414721555731e..8bcd6d2e9951b9 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -17,7 +17,7 @@ from datetime import date, datetime, time, timedelta, timezone from functools import cached_property -from test.support import MISSING_C_DOCSTRINGS +from test.support import MISSING_C_DOCSTRINGS, requires_gil_enabled from test.test_zoneinfo import _support as test_support from test.test_zoneinfo._support import OS_ENV_LOCK, TZPATH_TEST_LOCK, ZoneInfoTestBase from test.support.import_helper import import_module, CleanImport @@ -1931,6 +1931,7 @@ def test_cache_location(self): self.assertFalse(hasattr(c_zoneinfo.ZoneInfo, "_weak_cache")) self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache")) + @requires_gil_enabled("gh-117783: types may be immortalized") def test_gc_tracked(self): import gc From a0193479475a047b223f64130867a63b672c8dc2 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 10 May 2024 23:42:34 +0300 Subject: [PATCH 051/903] gh-118924: Remove `sqlite3.version` and `sqlite3.version_info` (#118925) --- Doc/library/sqlite3.rst | 22 +--------- Doc/whatsnew/3.13.rst | 2 +- Doc/whatsnew/3.14.rst | 40 ++++++++++++++----- Lib/sqlite3/__init__.py | 13 ------ Lib/sqlite3/dbapi2.py | 14 +------ Lib/test/test_sqlite3/test_dbapi.py | 11 ----- Misc/NEWS.d/3.12.0a1.rst | 2 +- ...-05-10-22-59-01.gh-issue-118924.9nyvSH.rst | 2 + Modules/_sqlite/module.c | 4 -- 9 files changed, 36 insertions(+), 74 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-10-22-59-01.gh-issue-118924.9nyvSH.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 6da8798ddfe0c0..5dc22a7e431ae4 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -545,26 +545,6 @@ Module constants .. versionchanged:: 3.11 Set *threadsafety* dynamically instead of hard-coding it to ``1``. -.. data:: version - - Version number of this module as a :class:`string `. - This is not the version of the SQLite library. - - .. deprecated-removed:: 3.12 3.14 - This constant used to reflect the version number of the ``pysqlite`` - package, a third-party library which used to upstream changes to - :mod:`!sqlite3`. Today, it carries no meaning or practical value. - -.. data:: version_info - - Version number of this module as a :class:`tuple` of :class:`integers `. - This is not the version of the SQLite library. - - .. deprecated-removed:: 3.12 3.14 - This constant used to reflect the version number of the ``pysqlite`` - package, a third-party library which used to upstream changes to - :mod:`!sqlite3`. Today, it carries no meaning or practical value. - .. _sqlite3-dbconfig-constants: .. data:: SQLITE_DBCONFIG_DEFENSIVE @@ -597,6 +577,8 @@ Module constants https://www.sqlite.org/c3ref/c_dbconfig_defensive.html SQLite docs: Database Connection Configuration Options +.. deprecated-removed:: 3.12 3.14 + The :data:`!version` and :data:`!version_info` constants. .. _sqlite3-connection-objects: diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 37c857dd8197e5..e69320e822ab3b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1636,7 +1636,7 @@ Pending Removal in Python 3.14 * :mod:`sqlite3`: - * :data:`~sqlite3.version` and :data:`~sqlite3.version_info`. + * :data:`!version` and :data:`!version_info`. * :meth:`~sqlite3.Cursor.execute` and :meth:`~sqlite3.Cursor.executemany` if :ref:`named placeholders ` are used and diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 52a24d1a9295a3..bcb1098f43d5a3 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -104,16 +104,30 @@ Removed argparse -------- -* The *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction` are removed. +* Remove the *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction`. They were deprecated since 3.12. +collections.abc +--------------- + +* Remove :class:`!collections.abc.ByteString`. It had previously raised a + :exc:`DeprecationWarning` since Python 3.12. + + email ----- -* The *isdst* parameter has been removed from :func:`email.utils.localtime`. +* Remove the *isdst* parameter from :func:`email.utils.localtime`. (Contributed by Hugo van Kemenade in :gh:`118798`.) +itertools +--------- + +* Remove :mod:`itertools` support for copy, deepcopy, and pickle operations. + These had previously raised a :exc:`DeprecationWarning` since Python 3.12. + (Contributed by Raymond Hettinger in :gh:`101588`.) + pathlib ------- @@ -122,6 +136,18 @@ pathlib :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such arguments are joined onto *other*. +sqlite3 +------- + +* Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. + (Contributed by Hugo van Kemenade in :gh:`118924`.) + +typing +------ + +* Remove :class:`!typing.ByteString`. It had previously raised a + :exc:`DeprecationWarning` since Python 3.12. + Others ------ @@ -129,14 +155,6 @@ Others It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed by Jelle Zijlstra in :gh:`118767`.) -* :class:`!typing.ByteString` and :class:`!collections.abc.ByteString` - are removed. They had previously raised a :exc:`DeprecationWarning` - since Python 3.12. - -* :mod:`itertools` support for copy, deepcopy, and pickle operations. - These had previously raised a :exc:`DeprecationWarning` since Python 3.12. - (Contributed by Raymond Hettinger in :gh:`101588`.) - Porting to Python 3.14 ====================== diff --git a/Lib/sqlite3/__init__.py b/Lib/sqlite3/__init__.py index 927267cf0b92ff..34a9c047dd607c 100644 --- a/Lib/sqlite3/__init__.py +++ b/Lib/sqlite3/__init__.py @@ -55,16 +55,3 @@ """ from sqlite3.dbapi2 import * -from sqlite3.dbapi2 import (_deprecated_names, - _deprecated_version_info, - _deprecated_version) - - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index 56fc0461e6c922..0315760516edf8 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -25,9 +25,6 @@ import collections.abc from _sqlite3 import * -from _sqlite3 import _deprecated_version - -_deprecated_names = frozenset({"version", "version_info"}) paramstyle = "qmark" @@ -48,7 +45,7 @@ def TimeFromTicks(ticks): def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) -_deprecated_version_info = tuple(map(int, _deprecated_version.split("."))) + sqlite_version_info = tuple([int(x) for x in sqlite_version.split(".")]) Binary = memoryview @@ -97,12 +94,3 @@ def convert_timestamp(val): # Clean up namespace del(register_adapters_and_converters) - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 51ce095df41fc1..1f71b5c34e448e 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -48,17 +48,6 @@ def test_api_level(self): self.assertEqual(sqlite.apilevel, "2.0", "apilevel is %s, should be 2.0" % sqlite.apilevel) - def test_deprecated_version(self): - msg = "deprecated and will be removed in Python 3.14" - for attr in "version", "version_info": - with self.subTest(attr=attr): - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite, attr) - self.assertEqual(cm.filename, __file__) - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite.dbapi2, attr) - self.assertEqual(cm.filename, __file__) - def test_thread_safety(self): self.assertIn(sqlite.threadsafety, {0, 1, 3}, "threadsafety is %d, should be 0, 1 or 3" % diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index f2438d6608b7af..1f259a64ee4494 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -3498,7 +3498,7 @@ Illia Volochii. .. nonce: tjfu9L .. section: Library -Deprecate :data:`sqlite3.version` and :data:`sqlite3.version_info`. +Deprecate :data:`!version` and :data:`!version_info`. .. diff --git a/Misc/NEWS.d/next/Library/2024-05-10-22-59-01.gh-issue-118924.9nyvSH.rst b/Misc/NEWS.d/next/Library/2024-05-10-22-59-01.gh-issue-118924.9nyvSH.rst new file mode 100644 index 00000000000000..36581dbb9bb11b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-10-22-59-01.gh-issue-118924.9nyvSH.rst @@ -0,0 +1,2 @@ +Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. +Patch by Hugo van Kemenade. diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 2c25ee32e58189..698e81d9b897d0 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -714,10 +714,6 @@ module_exec(PyObject *module) goto error; } - if (PyModule_AddStringConstant(module, "_deprecated_version", PYSQLITE_VERSION) < 0) { - goto error; - } - if (PyModule_AddStringConstant(module, "sqlite_version", sqlite3_libversion())) { goto error; } From b88889e9ffd7b2d2bdac75aecbf14e37fd68e337 Mon Sep 17 00:00:00 2001 From: mpage Date: Fri, 10 May 2024 14:54:23 -0700 Subject: [PATCH 052/903] gh-117657: Log TSAN warnings to separate files and archive them (#118747) This ensures we don't lose races that occur in subprocesses or interleave races from workers running in parallel. Log files are collected and packaged into a zipfile that can be downloaded from the "Artifacts" section of the workflow run. --- .github/workflows/build.yml | 2 ++ .github/workflows/reusable-tsan.yml | 16 +++++++++++++++- Tools/tsan/supressions.txt | 3 +++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a53f1ae1a46fc1..d14d17a5e088b6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -486,6 +486,7 @@ jobs: config_hash: ${{ needs.check_source.outputs.config_hash }} options: ./configure --config-cache --with-thread-sanitizer --with-pydebug suppressions_path: Tools/tsan/supressions.txt + tsan_logs_artifact_name: tsan-logs-default build_tsan_free_threading: name: 'Thread sanitizer (free-threading)' @@ -496,6 +497,7 @@ jobs: config_hash: ${{ needs.check_source.outputs.config_hash }} options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug suppressions_path: Tools/tsan/suppressions_free_threading.txt + tsan_logs_artifact_name: tsan-logs-free-threading # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 48bd5b547e8cba..b6d5d8fa1c7157 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -11,6 +11,10 @@ on: description: 'A repo relative path to the suppressions file' required: true type: string + tsan_logs_artifact_name: + description: 'Name of the TSAN logs artifact. Must be unique for each job.' + required: true + type: string jobs: build_tsan_reusable: @@ -41,7 +45,7 @@ jobs: sudo sysctl -w vm.mmap_rnd_bits=28 - name: TSAN Option Setup run: | - echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }}" >> $GITHUB_ENV + echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }} handle_segv=0" >> $GITHUB_ENV echo "CC=clang" >> $GITHUB_ENV echo "CXX=clang++" >> $GITHUB_ENV - name: Add ccache to PATH @@ -60,3 +64,13 @@ jobs: run: make pythoninfo - name: Tests run: ./python -m test --tsan -j4 + - name: Display TSAN logs + if: always() + run: find ${GITHUB_WORKSPACE} -name 'tsan_log.*' | xargs head -n 1000 + - name: Archive TSAN logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.tsan_logs_artifact_name }} + path: tsan_log.* + if-no-files-found: ignore diff --git a/Tools/tsan/supressions.txt b/Tools/tsan/supressions.txt index c778c791eacce8..22ba9d6ba2ab4d 100644 --- a/Tools/tsan/supressions.txt +++ b/Tools/tsan/supressions.txt @@ -2,3 +2,6 @@ # reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions race:get_allocator_unlocked race:set_allocator_unlocked + +# https://gist.github.com/mpage/daaf32b39180c1989572957b943eb665 +thread:pthread_create From 35c436186b849f8f2f9fb866c59015c9d034d448 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 10 May 2024 15:53:10 -0700 Subject: [PATCH 053/903] gh-118921: Add `copy()` method for `FrameLocalsProxy` (#118923) --- Lib/test/test_frame.py | 12 +++++++++--- ...-05-10-19-54-18.gh-issue-118921.O4ztZG.rst | 1 + Objects/frameobject.c | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-10-19-54-18.gh-issue-118921.O4ztZG.rst diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 212255374bddd1..aee8d374b22710 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -371,6 +371,15 @@ def test_local_objects(self): f_locals['o'] = f_locals['k'] self.assertEqual(o, 'a.b.c') + def test_copy(self): + x = 0 + d = sys._getframe().f_locals + d_copy = d.copy() + self.assertIsInstance(d_copy, dict) + self.assertEqual(d_copy['x'], 0) + d_copy['x'] = 1 + self.assertEqual(x, 0) + def test_update_with_self(self): def f(): f_locals = sys._getframe().f_locals @@ -405,9 +414,6 @@ def test_sizeof(self): def test_unsupport(self): x = 1 d = sys._getframe().f_locals - with self.assertRaises(AttributeError): - d.copy() - with self.assertRaises(TypeError): copy.copy(d) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-10-19-54-18.gh-issue-118921.O4ztZG.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-10-19-54-18.gh-issue-118921.O4ztZG.rst new file mode 100644 index 00000000000000..39ccf472067cfd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-10-19-54-18.gh-issue-118921.O4ztZG.rst @@ -0,0 +1 @@ +Add ``copy()`` method for ``FrameLocalsProxy`` which returns a snapshot ``dict`` for local variables. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d7fcb1925d286c..64fded85de1468 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -637,6 +637,23 @@ framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t na return result; } +static PyObject* +framelocalsproxy_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject* result = PyDict_New(); + + if (result == NULL) { + return NULL; + } + + if (PyDict_Update(result, self) < 0) { + Py_DECREF(result); + return NULL; + } + + return result; +} + static PyObject* framelocalsproxy_reversed(PyObject *self, void *Py_UNUSED(ignored)) { @@ -677,6 +694,8 @@ static PyMethodDef framelocalsproxy_methods[] = { NULL}, {"__reversed__", _PyCFunction_CAST(framelocalsproxy_reversed), METH_NOARGS, NULL}, + {"copy", _PyCFunction_CAST(framelocalsproxy_copy), METH_NOARGS, + NULL}, {"keys", _PyCFunction_CAST(framelocalsproxy_keys), METH_NOARGS, NULL}, {"values", _PyCFunction_CAST(framelocalsproxy_values), METH_NOARGS, From 7e894c2f38f64aed9b259c8fd31880f1142a259d Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Sat, 11 May 2024 13:47:45 +0200 Subject: [PATCH 054/903] Docs: Fix SOURCE_URI (#118945) --- Doc/tools/extensions/pyspecific.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index b35bedcb0901f1..44db77af5d24d3 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -33,7 +33,7 @@ ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' GH_ISSUE_URI = 'https://github.com/python/cpython/issues/%s' # Used in conf.py and updated here by python/release-tools/run_release.py -SOURCE_URI = 'https://github.com/python/cpython/tree/3.13/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/main/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body From cd4cfa6ed2fd5f866c7be339f1d3cf56aa4d2bad Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 11 May 2024 23:55:23 +0300 Subject: [PATCH 055/903] gh-118932: ChainMap.__contains__ performance improvement (gh-118946) --- Lib/collections/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index d06d84cbdfcc36..a17100e6c02a0e 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -1016,7 +1016,7 @@ def __getitem__(self, key): return self.__missing__(key) # support subclasses that define __missing__ def get(self, key, default=None): - return self[key] if key in self else default + return self[key] if key in self else default # needs to make use of __contains__ def __len__(self): return len(set().union(*self.maps)) # reuses stored hash values if possible @@ -1028,7 +1028,10 @@ def __iter__(self): return iter(d) def __contains__(self, key): - return any(key in m for m in self.maps) + for mapping in self.maps: + if key in mapping: + return True + return False def __bool__(self): return any(self.maps) From abead548af0172dabba13da8bacf2da3c02d4927 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Sat, 11 May 2024 23:46:07 +0200 Subject: [PATCH 056/903] gh-117655: Prevent `test_strptime` from raising a DeprecationWarning (GH-117668) * Fix `test_strptime` raises a DeprecationWarning * Ignore deprecation warnings where appropriate. * Update Lib/test/datetimetester.py This is follow on work to silence unnecessary warnings from the test suite that changes for https://github.com/python/cpython/issues/70647 added. --- Lib/test/datetimetester.py | 2 ++ Lib/test/test_strptime.py | 32 ++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 570110893629cf..b3838d5b406e94 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -22,6 +22,7 @@ from test import support from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST +from test.support import warnings_helper import datetime as datetime_module from datetime import MINYEAR, MAXYEAR @@ -2797,6 +2798,7 @@ def test_strptime_single_digit(self): newdate = strptime(string, format) self.assertEqual(newdate, target, msg=reason) + @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_strptime_leap_year(self): # GH-70647: warns if parsing a format with a day and no year. with self.assertRaises(ValueError): diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 05c8afc907ad3c..038746e26c24ad 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -7,7 +7,7 @@ import os import sys from test import support -from test.support import skip_if_buggy_ucrt_strfptime +from test.support import skip_if_buggy_ucrt_strfptime, warnings_helper from datetime import date as datetime_date import _strptime @@ -120,7 +120,7 @@ def setUp(self): def test_pattern(self): # Test TimeRE.pattern - pattern_string = self.time_re.pattern(r"%a %A %d") + pattern_string = self.time_re.pattern(r"%a %A %d %Y") self.assertTrue(pattern_string.find(self.locale_time.a_weekday[2]) != -1, "did not find abbreviated weekday in pattern string '%s'" % pattern_string) @@ -160,10 +160,11 @@ def test_compile(self): found.group('b'))) for directive in ('a','A','b','B','c','d','G','H','I','j','m','M','p', 'S','u','U','V','w','W','x','X','y','Y','Z','%'): - compiled = self.time_re.compile("%" + directive) - found = compiled.match(time.strftime("%" + directive)) + fmt = "%d %Y" if directive == 'd' else "%" + directive + compiled = self.time_re.compile(fmt) + found = compiled.match(time.strftime(fmt)) self.assertTrue(found, "Matching failed on '%s' using '%s' regex" % - (time.strftime("%" + directive), + (time.strftime(fmt), compiled.pattern)) def test_blankpattern(self): @@ -290,8 +291,9 @@ def test_unconverteddata(self): def helper(self, directive, position): """Helper fxn in testing.""" - strf_output = time.strftime("%" + directive, self.time_tuple) - strp_output = _strptime._strptime_time(strf_output, "%" + directive) + fmt = "%d %Y" if directive == 'd' else "%" + directive + strf_output = time.strftime(fmt, self.time_tuple) + strp_output = _strptime._strptime_time(strf_output, fmt) self.assertTrue(strp_output[position] == self.time_tuple[position], "testing of '%s' directive failed; '%s' -> %s != %s" % (directive, strf_output, strp_output[position], @@ -497,9 +499,11 @@ def test_escaping(self): need_escaping = r".^$*+?{}\[]|)(" self.assertTrue(_strptime._strptime_time(need_escaping, need_escaping)) + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-70647 def test_feb29_on_leap_year_without_year(self): time.strptime("Feb 29", "%b %d") + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-70647 def test_mar1_comes_after_feb29_even_when_omitting_the_year(self): self.assertLess( time.strptime("Feb 29", "%b %d"), @@ -679,25 +683,25 @@ class CacheTests(unittest.TestCase): def test_time_re_recreation(self): # Make sure cache is recreated when current locale does not match what # cached object was created with. - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") _strptime._strptime_time("2005", "%Y") _strptime._TimeRE_cache.locale_time.lang = "Ni" original_time_re = _strptime._TimeRE_cache - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertIsNot(original_time_re, _strptime._TimeRE_cache) self.assertEqual(len(_strptime._regex_cache), 1) def test_regex_cleanup(self): # Make sure cached regexes are discarded when cache becomes "full". try: - del _strptime._regex_cache['%d'] + del _strptime._regex_cache['%d %Y'] except KeyError: pass bogus_key = 0 while len(_strptime._regex_cache) <= _strptime._CACHE_MAX_SIZE: _strptime._regex_cache[bogus_key] = None bogus_key += 1 - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertEqual(len(_strptime._regex_cache), 1) def test_new_localetime(self): @@ -705,7 +709,7 @@ def test_new_localetime(self): # is created. locale_time_id = _strptime._TimeRE_cache.locale_time _strptime._TimeRE_cache.locale_time.lang = "Ni" - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertIsNot(locale_time_id, _strptime._TimeRE_cache.locale_time) def test_TimeRE_recreation_locale(self): @@ -716,13 +720,13 @@ def test_TimeRE_recreation_locale(self): except locale.Error: self.skipTest('test needs en_US.UTF8 locale') try: - _strptime._strptime_time('10', '%d') + _strptime._strptime_time('10 2004', '%d %Y') # Get id of current cache object. first_time_re = _strptime._TimeRE_cache try: # Change the locale and force a recreation of the cache. locale.setlocale(locale.LC_TIME, ('de_DE', 'UTF8')) - _strptime._strptime_time('10', '%d') + _strptime._strptime_time('10 2004', '%d %Y') # Get the new cache object's id. second_time_re = _strptime._TimeRE_cache # They should not be equal. From 5b941e57c71d7d0ab983d81a169f892662cfe446 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 11 May 2024 16:19:31 -0700 Subject: [PATCH 057/903] GH-118844: Fix build failures when combining --disable-gil with --enable-experimental-jit (GH-118935) --- Lib/subprocess.py | 2 +- .../2024-05-11-21-44-17.gh-issue-118844.q2H_km.rst | 1 + Python/jit.c | 1 + Python/perf_jit_trampoline.c | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-11-21-44-17.gh-issue-118844.q2H_km.rst diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 212fdf5b095511..b2dcb1454c139e 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -842,7 +842,7 @@ def __init__(self, args, bufsize=-1, executable=None, raise TypeError("bufsize must be an integer") if stdout is STDOUT: - raise ValueError("STDOUT can only be used for stderr") + raise ValueError("STDOUT can only be used for stderr") if pipesize is None: pipesize = -1 # Restore default diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-11-21-44-17.gh-issue-118844.q2H_km.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-11-21-44-17.gh-issue-118844.q2H_km.rst new file mode 100644 index 00000000000000..6e80b773889413 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-11-21-44-17.gh-issue-118844.q2H_km.rst @@ -0,0 +1 @@ +Fix build failures when configuring with both ``--disable-gil`` and ``--enable-experimental-jit``. diff --git a/Python/jit.c b/Python/jit.c index 7c316a410dda6a..d0c0d24f4539e2 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -5,6 +5,7 @@ #include "pycore_abstract.h" #include "pycore_call.h" #include "pycore_ceval.h" +#include "pycore_critical_section.h" #include "pycore_dict.h" #include "pycore_intrinsics.h" #include "pycore_long.h" diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 2a29318b1054a5..0a8945958b4b3c 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -174,8 +174,8 @@ static const uint8_t DwarfDataRel = 0x30; typedef struct { unsigned char version; unsigned char eh_frame_ptr_enc; - unsigned char fde_count_enc; - unsigned char table_enc; + unsigned char fde_count_enc; + unsigned char table_enc; int32_t eh_frame_ptr; int32_t eh_fde_count; int32_t from; From ec1398e117fb142cc830495503dbdbb1ddafe941 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 12 May 2024 17:00:49 +0300 Subject: [PATCH 058/903] gh-118899: Add tests for `NotImplemented` attribute access (#118902) --- Lib/test/test_builtin.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index a7631f92e7ea81..d7ba58847a2992 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2138,6 +2138,24 @@ def test_bool_notimplemented(self): with self.assertRaisesRegex(TypeError, msg): not NotImplemented + def test_singleton_attribute_access(self): + for singleton in (NotImplemented, Ellipsis): + with self.subTest(singleton): + self.assertIs(type(singleton), singleton.__class__) + self.assertIs(type(singleton).__class__, type) + + # Missing instance attributes: + with self.assertRaises(AttributeError): + singleton.prop = 1 + with self.assertRaises(AttributeError): + singleton.prop + + # Missing class attributes: + with self.assertRaises(TypeError): + type(singleton).prop = 1 + with self.assertRaises(AttributeError): + type(singleton).prop + class TestBreakpoint(unittest.TestCase): def setUp(self): From a705c1e44984afda2f7fb6d0816345d0843a6635 Mon Sep 17 00:00:00 2001 From: pochmann3 <150468338+pochmann3@users.noreply.github.com> Date: Sun, 12 May 2024 23:11:38 +0200 Subject: [PATCH 059/903] Itertools docs: fix parameter names and indentation in Python equivalents (gh-118977) --- Doc/library/itertools.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 9a1ed3594e81e3..afb7a6e835a220 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -380,7 +380,7 @@ loops that truncate the stream. saved.append(element) while saved: for element in saved: - yield element + yield element Note, this member of the toolkit may require significant auxiliary storage (depending on the length of the iterable). @@ -615,10 +615,10 @@ loops that truncate the stream. This function is roughly equivalent to the following code, except that the actual implementation does not build up intermediate results in memory:: - def product(*args, repeat=1): + def product(*iterables, repeat=1): # product('ABCD', 'xy') → Ax Ay Bx By Cx Cy Dx Dy # product(range(2), repeat=3) → 000 001 010 011 100 101 110 111 - pools = [tuple(pool) for pool in args] * repeat + pools = [tuple(pool) for pool in iterables] * repeat result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] @@ -735,9 +735,9 @@ loops that truncate the stream. iterables are of uneven length, missing values are filled-in with *fillvalue*. Iteration continues until the longest iterable is exhausted. Roughly equivalent to:: - def zip_longest(*args, fillvalue=None): + def zip_longest(*iterables, fillvalue=None): # zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D- - iterators = [iter(it) for it in args] + iterators = [iter(it) for it in iterables] num_active = len(iterators) if not num_active: return From 9c1520244151f36e010c1b04bedf14747a28517d Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Mon, 13 May 2024 03:56:09 -0400 Subject: [PATCH 060/903] gh-87106: Fix inspect.signature.bind() handling of positional-only arguments with **kwargs (GH-103404) --- Lib/inspect.py | 28 +++++++++++-------- Lib/test/test_inspect/test_inspect.py | 25 +++++++++++++---- ...3-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst | 3 ++ 3 files changed, 39 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 84260b251a4fb8..e6e49a4ffa673a 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -3106,6 +3106,8 @@ def _bind(self, args, kwargs, *, partial=False): parameters_ex = () arg_vals = iter(args) + pos_only_param_in_kwargs = [] + while True: # Let's iterate through the positional arguments and corresponding # parameters @@ -3126,10 +3128,10 @@ def _bind(self, args, kwargs, *, partial=False): break elif param.name in kwargs: if param.kind == _POSITIONAL_ONLY: - msg = '{arg!r} parameter is positional only, ' \ - 'but was passed as a keyword' - msg = msg.format(arg=param.name) - raise TypeError(msg) from None + # Raise a TypeError once we are sure there is no + # **kwargs param later. + pos_only_param_in_kwargs.append(param) + continue parameters_ex = (param,) break elif (param.kind == _VAR_KEYWORD or @@ -3211,20 +3213,22 @@ def _bind(self, args, kwargs, *, partial=False): format(arg=param_name)) from None else: - if param.kind == _POSITIONAL_ONLY: - # This should never happen in case of a properly built - # Signature object (but let's have this check here - # to ensure correct behaviour just in case) - raise TypeError('{arg!r} parameter is positional only, ' - 'but was passed as a keyword'. \ - format(arg=param.name)) - arguments[param_name] = arg_val if kwargs: if kwargs_param is not None: # Process our '**kwargs'-like parameter arguments[kwargs_param.name] = kwargs + elif pos_only_param_in_kwargs: + raise TypeError( + 'got some positional-only arguments passed as ' + 'keyword arguments: {arg!r}'.format( + arg=', '.join( + param.name + for param in pos_only_param_in_kwargs + ), + ), + ) else: raise TypeError( 'got an unexpected keyword argument {arg!r}'.format( diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 8bd13033490b81..011d42f34b6461 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -5089,15 +5089,30 @@ def test(a_po, b_po, c_po=3, /, foo=42, *, bar=50, **kwargs): self.assertEqual(self.call(test, 1, 2, foo=4, bar=5), (1, 2, 3, 4, 5, {})) - with self.assertRaisesRegex(TypeError, "but was passed as a keyword"): - self.call(test, 1, 2, foo=4, bar=5, c_po=10) + self.assertEqual(self.call(test, 1, 2, foo=4, bar=5, c_po=10), + (1, 2, 3, 4, 5, {'c_po': 10})) - with self.assertRaisesRegex(TypeError, "parameter is positional only"): - self.call(test, 1, 2, c_po=4) + self.assertEqual(self.call(test, 1, 2, 30, c_po=31, foo=4, bar=5), + (1, 2, 30, 4, 5, {'c_po': 31})) - with self.assertRaisesRegex(TypeError, "parameter is positional only"): + self.assertEqual(self.call(test, 1, 2, 30, foo=4, bar=5, c_po=31), + (1, 2, 30, 4, 5, {'c_po': 31})) + + self.assertEqual(self.call(test, 1, 2, c_po=4), + (1, 2, 3, 42, 50, {'c_po': 4})) + + with self.assertRaisesRegex(TypeError, "missing 2 required positional arguments"): self.call(test, a_po=1, b_po=2) + def without_var_kwargs(c_po=3, d_po=4, /): + return c_po, d_po + + with self.assertRaisesRegex( + TypeError, + "positional-only arguments passed as keyword arguments: 'c_po, d_po'", + ): + self.call(without_var_kwargs, c_po=33, d_po=44) + def test_signature_bind_with_self_arg(self): # Issue #17071: one of the parameters is named "self def test(a, self, b): diff --git a/Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst b/Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst new file mode 100644 index 00000000000000..6f13188f6f119f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst @@ -0,0 +1,3 @@ +Fixed handling in :meth:`inspect.signature.bind` of keyword arguments having +the same name as positional-only arguments when a variadic keyword argument +(e.g. ``**kwargs``) is present. From b4ca389281849e849fb58fecf9b31e2e2f5a39c1 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 13 May 2024 14:04:14 +0300 Subject: [PATCH 061/903] Improve the `rmtree` doc for `dir_fd` param addition in 3.11 (#118964) --- Doc/library/shutil.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 1a053c32866153..fd32479195eca8 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -338,7 +338,7 @@ Directory and files operations before removing the junction. .. versionchanged:: 3.11 - The *dir_fd* parameter. + Added the *dir_fd* parameter. .. versionchanged:: 3.12 Added the *onexc* parameter, deprecated *onerror*. From d8a82cca12e12a6b22bfe6691e9b222f6d276f0a Mon Sep 17 00:00:00 2001 From: I-Shen Leong Date: Mon, 13 May 2024 04:30:16 -0700 Subject: [PATCH 062/903] gh-118876: Ensure PC/layout sets ns.temp before using it (GH-118880) Fixes an AttributeError that occurs when checking if ns.temp is an absolute path during building from source on Windows. --- PC/layout/main.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/PC/layout/main.py b/PC/layout/main.py index d176b272f1c19d..1c4842f8588a5b 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -599,6 +599,15 @@ def main(): ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) ns.build = ns.build or Path(sys.executable).parent ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") + if ns.copy and not ns.copy.is_absolute(): + ns.copy = (Path.cwd() / ns.copy).resolve() + if not ns.temp: + # Put temp on a Dev Drive for speed if we're copying to one. + # If not, the regular temp dir will have to do. + if ns.copy and getattr(os.path, "isdevdrive", lambda d: False)(ns.copy): + ns.temp = ns.copy.with_name(ns.copy.name + "_temp") + else: + ns.temp = Path(tempfile.mkdtemp()) if not ns.source.is_absolute(): ns.source = (Path.cwd() / ns.source).resolve() if not ns.build.is_absolute(): @@ -617,21 +626,11 @@ def main(): else: ns.arch = "amd64" - if ns.copy and not ns.copy.is_absolute(): - ns.copy = (Path.cwd() / ns.copy).resolve() if ns.zip and not ns.zip.is_absolute(): ns.zip = (Path.cwd() / ns.zip).resolve() if ns.catalog and not ns.catalog.is_absolute(): ns.catalog = (Path.cwd() / ns.catalog).resolve() - if not ns.temp: - # Put temp on a Dev Drive for speed if we're copying to one. - # If not, the regular temp dir will have to do. - if ns.copy and getattr(os.path, "isdevdrive", lambda d: False)(ns.copy): - ns.temp = ns.copy.with_name(ns.copy.name + "_temp") - else: - ns.temp = Path(tempfile.mkdtemp()) - configure_logger(ns) log_info( From f526314194f7fd15931025f8a4439c1765666e42 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 13 May 2024 05:38:21 -0700 Subject: [PATCH 063/903] gh-58933: Make pdb return to caller frame correctly when f_trace is not set (#118979) --- Lib/bdb.py | 16 +++--- Lib/test/test_pdb.py | 52 +++++++++++++++++++ ...4-05-12-21-38-42.gh-issue-58933.0kgU2l.rst | 1 + 3 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-12-21-38-42.gh-issue-58933.0kgU2l.rst diff --git a/Lib/bdb.py b/Lib/bdb.py index 7d63fce6ca63f1..aa621053cfb4bc 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -165,6 +165,11 @@ def dispatch_return(self, frame, arg): # The user issued a 'next' or 'until' command. if self.stopframe is frame and self.stoplineno != -1: self._set_stopinfo(None, None) + # The previous frame might not have f_trace set, unless we are + # issuing a command that does not expect to stop, we should set + # f_trace + if self.stoplineno != -1: + self._set_caller_tracefunc(frame) return self.trace_dispatch def dispatch_exception(self, frame, arg): @@ -320,15 +325,14 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): self.stoplineno = stoplineno self._set_trace_opcodes(opcode) - def _set_caller_tracefunc(self): + def _set_caller_tracefunc(self, current_frame): # Issue #13183: pdb skips frames after hitting a breakpoint and running # step commands. # Restore the trace function in the caller (that may not have been set # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and not caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch + caller_frame = current_frame.f_back + if caller_frame and not caller_frame.f_trace: + caller_frame.f_trace = self.trace_dispatch # Derived classes and clients can call the following methods # to affect the stepping state. @@ -343,12 +347,10 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - self._set_caller_tracefunc() self._set_stopinfo(None, None) def set_stepinstr(self): """Stop before the next instruction.""" - self._set_caller_tracefunc() self._set_stopinfo(None, None, opcode=True) def set_next(self, frame): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index f47466410082ef..1b329b205d2d0f 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1453,6 +1453,58 @@ def test_post_mortem(): """ +def test_pdb_return_to_different_file(): + """When pdb returns to a different file, it should not skip if f_trace is + not already set + + >>> import pprint + + >>> class A: + ... def __repr__(self): + ... return 'A' + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... pprint.pprint(A()) + + >>> reset_Breakpoint() + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'b A.__repr__', + ... 'continue', + ... 'return', + ... 'next', + ... 'return', + ... 'return', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) b A.__repr__ + Breakpoint 1 at :3 + (Pdb) continue + > (3)__repr__() + -> return 'A' + (Pdb) return + --Return-- + > (3)__repr__()->'A' + -> return 'A' + (Pdb) next + > ...pprint.py..._safe_repr() + -> return rep,... + (Pdb) return + --Return-- + > ...pprint.py..._safe_repr()->('A'...) + -> return rep,... + (Pdb) return + --Return-- + > ...pprint.py...format()->('A'...) + -> return... + (Pdb) continue + A + """ + + def test_pdb_skip_modules(): """This illustrates the simple case of module skipping. diff --git a/Misc/NEWS.d/next/Library/2024-05-12-21-38-42.gh-issue-58933.0kgU2l.rst b/Misc/NEWS.d/next/Library/2024-05-12-21-38-42.gh-issue-58933.0kgU2l.rst new file mode 100644 index 00000000000000..fa70b954e1e9ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-12-21-38-42.gh-issue-58933.0kgU2l.rst @@ -0,0 +1 @@ +Make :mod:`pdb` return to caller frame correctly when ``f_trace`` of the caller frame is not set From 7d7eec595a47a5cd67ab420164f0059eb8b9aa28 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 13 May 2024 16:03:52 +0200 Subject: [PATCH 064/903] gh-117873: Revert _posixshmem.shm_open() change (#118901) --- Modules/_multiprocessing/clinic/posixshmem.c.h | 6 +++--- Modules/_multiprocessing/posixshmem.c | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 8151f2e0b07082..a545ff4d80f067 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -5,7 +5,7 @@ preserve #if defined(HAVE_SHM_OPEN) PyDoc_STRVAR(_posixshmem_shm_open__doc__, -"shm_open($module, path, /, flags, mode=511)\n" +"shm_open($module, /, path, flags, mode=511)\n" "--\n" "\n" "Open a shared memory object. Returns a file descriptor (integer)."); @@ -21,7 +21,7 @@ static PyObject * _posixshmem_shm_open(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - static char *_keywords[] = {"", "flags", "mode", NULL}; + static char *_keywords[] = {"path", "flags", "mode", NULL}; PyObject *path; int flags; int mode = 511; @@ -86,4 +86,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *arg) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=649877fc45a65129 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=74588a5abba6e36c input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index cc157800ade3c4..ab45e4136c7d46 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -32,7 +32,6 @@ module _posixshmem /*[clinic input] _posixshmem.shm_open -> int path: unicode - / flags: int mode: int = 0o777 @@ -45,7 +44,7 @@ Open a shared memory object. Returns a file descriptor (integer). static int _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, int mode) -/*[clinic end generated code: output=8d110171a4fa20df input=0585935e1d3c8050]*/ +/*[clinic end generated code: output=8d110171a4fa20df input=e83b58fa802fac25]*/ { int fd; int async_err = 0; From b04c497f187b0b474e431a6d8d282269b40ffe52 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 13 May 2024 23:01:05 +0300 Subject: [PATCH 065/903] gh-119010: Adds docs about `__type_params__` to `functools.update_wrapper` (#119012) Co-authored-by: Jelle Zijlstra --- Doc/library/functools.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 1f003b3a94fda2..9d5c72802a21f2 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -646,8 +646,9 @@ The :mod:`functools` module defines the following functions: attributes of the wrapper function are updated with the corresponding attributes from the original function. The default values for these arguments are the module level constants ``WRAPPER_ASSIGNMENTS`` (which assigns to the wrapper - function's ``__module__``, ``__name__``, ``__qualname__``, ``__annotations__`` - and ``__doc__``, the documentation string) and ``WRAPPER_UPDATES`` (which + function's ``__module__``, ``__name__``, ``__qualname__``, ``__annotations__``, + ``__type_params__``, and ``__doc__``, the documentation string) + and ``WRAPPER_UPDATES`` (which updates the wrapper function's ``__dict__``, i.e. the instance dictionary). To allow access to the original function for introspection and other purposes @@ -677,6 +678,9 @@ The :mod:`functools` module defines the following functions: function, even if that function defined a ``__wrapped__`` attribute. (see :issue:`17482`) + .. versionchanged:: 3.12 + The ``__type_params__`` attribute is now copied by default. + .. decorator:: wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) From fc757925944a9486d4244853dbe6e37ab3e560c2 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 14 May 2024 00:20:59 +0300 Subject: [PATCH 066/903] gh-118998: Handle errors correctly in `tmtotuple` in `timemodule` (#118999) --- Modules/timemodule.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 0511339978897a..ed2d32688ecea5 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -462,7 +462,18 @@ tmtotuple(time_module_state *state, struct tm *p if (v == NULL) return NULL; -#define SET(i,val) PyStructSequence_SET_ITEM(v, i, PyLong_FromLong((long) val)) +#define SET_ITEM(INDEX, CALL) \ + do { \ + PyObject *obj = (CALL); \ + if (obj == NULL) { \ + Py_DECREF(v); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(v, (INDEX), obj); \ + } while (0) + +#define SET(INDEX, VAL) \ + SET_ITEM((INDEX), PyLong_FromLong((long) (VAL))) SET(0, p->tm_year + 1900); SET(1, p->tm_mon + 1); /* Want January == 1 */ @@ -474,19 +485,15 @@ tmtotuple(time_module_state *state, struct tm *p SET(7, p->tm_yday + 1); /* Want January, 1 == 1 */ SET(8, p->tm_isdst); #ifdef HAVE_STRUCT_TM_TM_ZONE - PyStructSequence_SET_ITEM(v, 9, - PyUnicode_DecodeLocale(p->tm_zone, "surrogateescape")); + SET_ITEM(9, PyUnicode_DecodeLocale(p->tm_zone, "surrogateescape")); SET(10, p->tm_gmtoff); #else - PyStructSequence_SET_ITEM(v, 9, - PyUnicode_DecodeLocale(zone, "surrogateescape")); - PyStructSequence_SET_ITEM(v, 10, _PyLong_FromTime_t(gmtoff)); + SET_ITEM(9, PyUnicode_DecodeLocale(zone, "surrogateescape")); + SET_ITEM(10, _PyLong_FromTime_t(gmtoff)); #endif /* HAVE_STRUCT_TM_TM_ZONE */ + #undef SET - if (PyErr_Occurred()) { - Py_XDECREF(v); - return NULL; - } +#undef SET_ITEM return v; } From e04cd964eb4eee1b0ae5b2c34727abce6c0fb7f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 13 May 2024 23:37:02 +0200 Subject: [PATCH 067/903] GH-118836: Fix JIT build error when SHT_NOTE section is present (GH-119000) --- .../next/Build/2024-05-13-15-57-58.gh-issue-118836.7yN1iB.rst | 2 ++ Tools/jit/_targets.py | 1 + 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2024-05-13-15-57-58.gh-issue-118836.7yN1iB.rst diff --git a/Misc/NEWS.d/next/Build/2024-05-13-15-57-58.gh-issue-118836.7yN1iB.rst b/Misc/NEWS.d/next/Build/2024-05-13-15-57-58.gh-issue-118836.7yN1iB.rst new file mode 100644 index 00000000000000..5212af7b32b940 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-13-15-57-58.gh-issue-118836.7yN1iB.rst @@ -0,0 +1,2 @@ +Fix an ``AssertionError`` when building with ``--enable-experimental-jit`` +and the compiler emits a ``SHT_NOTE`` section. diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 023ef498a21d7c..b020f49cf4a2c1 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -349,6 +349,7 @@ def _handle_section( assert section_type in { "SHT_GROUP", "SHT_LLVM_ADDRSIG", + "SHT_NOTE", "SHT_NULL", "SHT_STRTAB", "SHT_SYMTAB", From e237b25a4fa5626fcd1b1848aa03f725f892e40e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 May 2024 12:24:37 +0300 Subject: [PATCH 068/903] gh-67693: Fix urlunparse() and urlunsplit() for URIs with path starting with multiple slashes and no authority (GH-113563) --- Lib/test/test_urlparse.py | 70 ++++++++++++++++++- Lib/urllib/parse.py | 2 +- ...9-08-27-01-16-50.gh-issue-67693.4NIAiy.rst | 2 + 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-08-27-01-16-50.gh-issue-67693.4NIAiy.rst diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 236b6e4516490a..2cf03d046a5b87 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -103,7 +103,9 @@ class UrlParseTestCase(unittest.TestCase): - def checkRoundtrips(self, url, parsed, split): + def checkRoundtrips(self, url, parsed, split, url2=None): + if url2 is None: + url2 = url result = urllib.parse.urlparse(url) self.assertSequenceEqual(result, parsed) t = (result.scheme, result.netloc, result.path, @@ -111,7 +113,7 @@ def checkRoundtrips(self, url, parsed, split): self.assertSequenceEqual(t, parsed) # put it back together and it should be the same result2 = urllib.parse.urlunparse(result) - self.assertSequenceEqual(result2, url) + self.assertSequenceEqual(result2, url2) self.assertSequenceEqual(result2, result.geturl()) # the result of geturl() is a fixpoint; we can always parse it @@ -137,7 +139,7 @@ def checkRoundtrips(self, url, parsed, split): result.query, result.fragment) self.assertSequenceEqual(t, split) result2 = urllib.parse.urlunsplit(result) - self.assertSequenceEqual(result2, url) + self.assertSequenceEqual(result2, url2) self.assertSequenceEqual(result2, result.geturl()) # check the fixpoint property of re-parsing the result of geturl() @@ -175,9 +177,39 @@ def test_qs(self): def test_roundtrips(self): str_cases = [ + ('path/to/file', + ('', '', 'path/to/file', '', '', ''), + ('', '', 'path/to/file', '', '')), + ('/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('//path/to/file', + ('', 'path', '/to/file', '', '', ''), + ('', 'path', '/to/file', '', '')), + ('////path/to/file', + ('', '', '//path/to/file', '', '', ''), + ('', '', '//path/to/file', '', '')), + ('scheme:path/to/file', + ('scheme', '', 'path/to/file', '', '', ''), + ('scheme', '', 'path/to/file', '', '')), + ('scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('scheme://path/to/file', + ('scheme', 'path', '/to/file', '', '', ''), + ('scheme', 'path', '/to/file', '', '')), + ('scheme:////path/to/file', + ('scheme', '', '//path/to/file', '', '', ''), + ('scheme', '', '//path/to/file', '', '')), ('file:///tmp/junk.txt', ('file', '', '/tmp/junk.txt', '', '', ''), ('file', '', '/tmp/junk.txt', '', '')), + ('file:////tmp/junk.txt', + ('file', '', '//tmp/junk.txt', '', '', ''), + ('file', '', '//tmp/junk.txt', '', '')), + ('file://///tmp/junk.txt', + ('file', '', '///tmp/junk.txt', '', '', ''), + ('file', '', '///tmp/junk.txt', '', '')), ('imap://mail.python.org/mbox1', ('imap', 'mail.python.org', '/mbox1', '', '', ''), ('imap', 'mail.python.org', '/mbox1', '', '')), @@ -213,6 +245,38 @@ def _encode(t): for url, parsed, split in str_cases + bytes_cases: self.checkRoundtrips(url, parsed, split) + def test_roundtrips_normalization(self): + str_cases = [ + ('///path/to/file', + '/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('scheme:///path/to/file', + 'scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('file:/tmp/junk.txt', + 'file:///tmp/junk.txt', + ('file', '', '/tmp/junk.txt', '', '', ''), + ('file', '', '/tmp/junk.txt', '', '')), + ('http:/tmp/junk.txt', + 'http:///tmp/junk.txt', + ('http', '', '/tmp/junk.txt', '', '', ''), + ('http', '', '/tmp/junk.txt', '', '')), + ('https:/tmp/junk.txt', + 'https:///tmp/junk.txt', + ('https', '', '/tmp/junk.txt', '', '', ''), + ('https', '', '/tmp/junk.txt', '', '')), + ] + def _encode(t): + return (t[0].encode('ascii'), + t[1].encode('ascii'), + tuple(x.encode('ascii') for x in t[2]), + tuple(x.encode('ascii') for x in t[3])) + bytes_cases = [_encode(x) for x in str_cases] + for url, url2, parsed, split in str_cases + bytes_cases: + self.checkRoundtrips(url, parsed, split, url2) + def test_http_roundtrips(self): # urllib.parse.urlsplit treats 'http:' as an optimized special case, # so we test both 'http:' and 'https:' in all the following. diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index fc9e7c99f283be..3932bb99c7e7d1 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -525,7 +525,7 @@ def urlunsplit(components): empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'): + if netloc or (scheme and scheme in uses_netloc) or url[:2] == '//': if url and url[:1] != '/': url = '/' + url url = '//' + (netloc or '') + url if scheme: diff --git a/Misc/NEWS.d/next/Library/2019-08-27-01-16-50.gh-issue-67693.4NIAiy.rst b/Misc/NEWS.d/next/Library/2019-08-27-01-16-50.gh-issue-67693.4NIAiy.rst new file mode 100644 index 00000000000000..22457df03e65c9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-08-27-01-16-50.gh-issue-67693.4NIAiy.rst @@ -0,0 +1,2 @@ +Fix :func:`urllib.parse.urlunparse` and :func:`urllib.parse.urlunsplit` for URIs with path starting with multiple slashes and no authority. +Based on patch by Ashwin Ramaswami. From 331d385af9817eaa32b739130227781358f85771 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 May 2024 16:59:21 +0300 Subject: [PATCH 069/903] Add yet few cases for urlparse/urlunparse roundtrip tests (GH-119031) Add yet few cases for urlparse/urlunparse tests --- Lib/test/test_urlparse.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 2cf03d046a5b87..4faad733245df9 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -189,6 +189,9 @@ def test_roundtrips(self): ('////path/to/file', ('', '', '//path/to/file', '', '', ''), ('', '', '//path/to/file', '', '')), + ('/////path/to/file', + ('', '', '///path/to/file', '', '', ''), + ('', '', '///path/to/file', '', '')), ('scheme:path/to/file', ('scheme', '', 'path/to/file', '', '', ''), ('scheme', '', 'path/to/file', '', '')), @@ -201,6 +204,9 @@ def test_roundtrips(self): ('scheme:////path/to/file', ('scheme', '', '//path/to/file', '', '', ''), ('scheme', '', '//path/to/file', '', '')), + ('scheme://///path/to/file', + ('scheme', '', '///path/to/file', '', '', ''), + ('scheme', '', '///path/to/file', '', '')), ('file:///tmp/junk.txt', ('file', '', '/tmp/junk.txt', '', '', ''), ('file', '', '/tmp/junk.txt', '', '')), @@ -236,12 +242,23 @@ def test_roundtrips(self): 'action=download-manifest&url=https://example.com/app', ''), ('itms-services', '', '', 'action=download-manifest&url=https://example.com/app', '')), + ('+scheme:path/to/file', + ('', '', '+scheme:path/to/file', '', '', ''), + ('', '', '+scheme:path/to/file', '', '')), + ('sch_me:path/to/file', + ('', '', 'sch_me:path/to/file', '', '', ''), + ('', '', 'sch_me:path/to/file', '', '')), ] def _encode(t): return (t[0].encode('ascii'), tuple(x.encode('ascii') for x in t[1]), tuple(x.encode('ascii') for x in t[2])) bytes_cases = [_encode(x) for x in str_cases] + str_cases += [ + ('schème:path/to/file', + ('', '', 'schème:path/to/file', '', '', ''), + ('', '', 'schème:path/to/file', '', '')), + ] for url, parsed, split in str_cases + bytes_cases: self.checkRoundtrips(url, parsed, split) From a9328e2b6ee05c186dcc552feb92b862b4a574df Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 14 May 2024 10:16:14 -0400 Subject: [PATCH 070/903] typing tests: remove some unnecessary uses of `exec()` (#119005) --- Lib/test/test_typing.py | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f10b0aea3cd7b9..64c4c497eb8934 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7060,24 +7060,16 @@ def test_iterator(self): self.assertNotIsInstance(42, typing.Iterator) def test_awaitable(self): - ns = {} - exec( - "async def foo() -> typing.Awaitable[int]:\n" - " return await AwaitableWrapper(42)\n", - globals(), ns) - foo = ns['foo'] + async def foo() -> typing.Awaitable[int]: + return await AwaitableWrapper(42) g = foo() self.assertIsInstance(g, typing.Awaitable) self.assertNotIsInstance(foo, typing.Awaitable) g.send(None) # Run foo() till completion, to avoid warning. def test_coroutine(self): - ns = {} - exec( - "async def foo():\n" - " return\n", - globals(), ns) - foo = ns['foo'] + async def foo(): + return g = foo() self.assertIsInstance(g, typing.Coroutine) with self.assertRaises(TypeError): @@ -7352,10 +7344,9 @@ def test_no_generator_instantiation(self): typing.Generator[int, int, int]() def test_async_generator(self): - ns = {} - exec("async def f():\n" - " yield 42\n", globals(), ns) - g = ns['f']() + async def f(): + yield 42 + g = f() self.assertIsSubclass(type(g), typing.AsyncGenerator) def test_no_async_generator_instantiation(self): @@ -7442,9 +7433,8 @@ def asend(self, value): def athrow(self, typ, val=None, tb=None): pass - ns = {} - exec('async def g(): yield 0', globals(), ns) - g = ns['g'] + async def g(): yield 0 + self.assertIsSubclass(G, typing.AsyncGenerator) self.assertIsSubclass(G, typing.AsyncIterable) self.assertIsSubclass(G, collections.abc.AsyncGenerator) From 7a97ee570f361af27c59eb883b53425fef11c739 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 14 May 2024 10:18:19 -0500 Subject: [PATCH 071/903] Misc improvements to the itertools docs (gh-119040) --- Doc/library/itertools.rst | 61 +++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index afb7a6e835a220..a19baa3f0e439f 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -122,15 +122,15 @@ loops that truncate the stream. # accumulate([1,2,3,4,5]) → 1 3 6 10 15 # accumulate([1,2,3,4,5], initial=100) → 100 101 103 106 110 115 # accumulate([1,2,3,4,5], operator.mul) → 1 2 6 24 120 - it = iter(iterable) + iterator = iter(iterable) total = initial if initial is None: try: - total = next(it) + total = next(iterator) except StopIteration: return yield total - for element in it: + for element in iterator: total = func(total, element) yield total @@ -218,9 +218,8 @@ loops that truncate the stream. def chain(*iterables): # chain('ABC', 'DEF') → A B C D E F - for it in iterables: - for element in it: - yield element + for iterable in iterables: + yield from iterable .. classmethod:: chain.from_iterable(iterable) @@ -230,9 +229,8 @@ loops that truncate the stream. def from_iterable(iterables): # chain.from_iterable(['ABC', 'DEF']) → A B C D E F - for it in iterables: - for element in it: - yield element + for iterable in iterables: + yield from iterable .. function:: combinations(iterable, r) @@ -696,24 +694,22 @@ loops that truncate the stream. Return *n* independent iterators from a single iterable. - The following Python code helps explain what *tee* does (although the actual - implementation is more complex and uses only a single underlying - :abbr:`FIFO (first-in, first-out)` queue):: + Roughly equivalent to:: def tee(iterable, n=2): - it = iter(iterable) - deques = [collections.deque() for i in range(n)] - def gen(mydeque): - while True: - if not mydeque: # when the local deque is empty - try: - newval = next(it) # fetch a new value and - except StopIteration: - return - for d in deques: # load it to all the deques - d.append(newval) - yield mydeque.popleft() - return tuple(gen(d) for d in deques) + iterator = iter(iterable) + empty_link = [None, None] # Singly linked list: [value, link] + return tuple(_tee(iterator, empty_link) for _ in range(n)) + + def _tee(iterator, link): + while True: + if link[1] is None: + try: + link[:] = [next(iterator), [None, None]] + except StopIteration: + return + value, link = link + yield value Once a :func:`tee` has been created, the original *iterable* should not be used anywhere else; otherwise, the *iterable* could get advanced without @@ -743,9 +739,9 @@ loops that truncate the stream. return while True: values = [] - for i, it in enumerate(iterators): + for i, iterator in enumerate(iterators): try: - value = next(it) + value = next(iterator) except StopIteration: num_active -= 1 if not num_active: @@ -800,6 +796,7 @@ and :term:`generators ` which incur interpreter overhead. .. testcode:: import collections + import contextlib import functools import math import operator @@ -942,32 +939,26 @@ and :term:`generators ` which incur interpreter overhead. # iter_index('AABCADEAF', 'A') → 0 1 4 7 seq_index = getattr(iterable, 'index', None) if seq_index is None: - # Path for general iterables iterator = islice(iterable, start, stop) for i, element in enumerate(iterator, start): if element is value or element == value: yield i else: - # Path for sequences with an index() method stop = len(iterable) if stop is None else stop i = start - try: + with contextlib.suppress(ValueError): while True: yield (i := seq_index(value, i, stop)) i += 1 - except ValueError: - pass def iter_except(func, exception, first=None): "Convert a call-until-exception interface to an iterator interface." # iter_except(d.popitem, KeyError) → non-blocking dictionary iterator - try: + with contextlib.suppress(exception): if first is not None: yield first() while True: yield func() - except exception: - pass The following recipes have a more mathematical flavor: From d8e0e009195b2388fb53012c1f0fa786426dc05f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 14 May 2024 12:10:55 -0400 Subject: [PATCH 072/903] gh-118928: sqlite3: disallow sequences of params with named placeholders (#118929) Follow-up of gh-101693. The previous DeprecationWarning is replaced with raising sqlite3.ProgrammingError. Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.14.rst | 5 +++++ Lib/test/test_sqlite3/test_dbapi.py | 3 +-- .../2024-05-10-22-36-01.gh-issue-118928.IW7Ukv.rst | 2 ++ Modules/_sqlite/cursor.c | 8 ++------ 4 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-10-22-36-01.gh-issue-118928.IW7Ukv.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index bcb1098f43d5a3..33a0f3e0f2f4bc 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -142,6 +142,11 @@ sqlite3 * Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. (Contributed by Hugo van Kemenade in :gh:`118924`.) +* Disallow using a sequence of parameters with named placeholders. + This had previously raised a :exc:`DeprecationWarning` since Python 3.12; + it will now raise a :exc:`sqlite3.ProgrammingError`. + (Contributed by Erlend E. Aasland in :gh:`118928` and :gh:`101693`.) + typing ------ diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 1f71b5c34e448e..293baccaf1831d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -878,9 +878,8 @@ def test_execute_named_param_and_sequence(self): msg = "Binding.*is a named parameter" for query, params in dataset: with self.subTest(query=query, params=params): - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: + with self.assertRaisesRegex(sqlite.ProgrammingError, msg) as cm: self.cu.execute(query, params) - self.assertEqual(cm.filename, __file__) def test_execute_indexed_nameless_params(self): # See gh-117995: "'?1' is considered a named placeholder" diff --git a/Misc/NEWS.d/next/Library/2024-05-10-22-36-01.gh-issue-118928.IW7Ukv.rst b/Misc/NEWS.d/next/Library/2024-05-10-22-36-01.gh-issue-118928.IW7Ukv.rst new file mode 100644 index 00000000000000..91c95e4a5395d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-10-22-36-01.gh-issue-118928.IW7Ukv.rst @@ -0,0 +1,2 @@ +Disallow using a sequence of parameters with named placeholders in +:mod:`sqlite3` queries. Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 950596ea82b568..5d4b77b1a07e08 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -670,15 +670,11 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, for (i = 0; i < num_params; i++) { const char *name = sqlite3_bind_parameter_name(self->st, i+1); if (name != NULL && name[0] != '?') { - int ret = PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + PyErr_Format(state->ProgrammingError, "Binding %d ('%s') is a named parameter, but you " "supplied a sequence which requires nameless (qmark) " - "placeholders. Starting with Python 3.14 an " - "sqlite3.ProgrammingError will be raised.", + "placeholders.", i+1, name); - if (ret < 0) { - return; - } } if (PyTuple_CheckExact(parameters)) { From fbe6a0988ff08aef29c4649527d5d620d77ca4a2 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Tue, 14 May 2024 18:53:15 +0100 Subject: [PATCH 073/903] GH-101357: Suppress `OSError` from `pathlib.Path.exists()` and `is_*()` (#118243) Suppress all `OSError` exceptions from `pathlib.Path.exists()` and `is_*()` rather than a selection of more common errors as we do presently. Also adjust the implementations to call `os.path.exists()` etc, which are much faster on Windows thanks to GH-101196. --- Doc/library/pathlib.rst | 75 ++++++++------- Lib/glob.py | 12 +-- Lib/pathlib/_abc.py | 91 ++----------------- Lib/pathlib/_local.py | 34 +++++++ ...-05-08-19-47-34.gh-issue-101357.e4R_9x.rst | 5 + 5 files changed, 92 insertions(+), 125 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 15e9eaa5190256..27ed0a32e801cc 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -873,7 +873,7 @@ Methods ^^^^^^^ Concrete paths provide the following methods in addition to pure paths -methods. Many of these methods can raise an :exc:`OSError` if a system +methods. Some of these methods can raise an :exc:`OSError` if a system call fails (for example because the path doesn't exist). .. versionchanged:: 3.8 @@ -885,6 +885,15 @@ call fails (for example because the path doesn't exist). instead of raising an exception for paths that contain characters unrepresentable at the OS level. +.. versionchanged:: 3.14 + + The methods given above now return ``False`` instead of raising any + :exc:`OSError` exception from the operating system. In previous versions, + some kinds of :exc:`OSError` exception are raised, and others suppressed. + The new behaviour is consistent with :func:`os.path.exists`, + :func:`os.path.isdir`, etc. Use :meth:`~Path.stat` to retrieve the file + status without suppressing exceptions. + .. classmethod:: Path.cwd() @@ -951,6 +960,8 @@ call fails (for example because the path doesn't exist). .. method:: Path.exists(*, follow_symlinks=True) Return ``True`` if the path points to an existing file or directory. + ``False`` will be returned if the path is invalid, inaccessible or missing. + Use :meth:`Path.stat` to distinguish between these cases. This method normally follows symlinks; to check if a symlink exists, add the argument ``follow_symlinks=False``. @@ -1067,11 +1078,10 @@ call fails (for example because the path doesn't exist). .. method:: Path.is_dir(*, follow_symlinks=True) - Return ``True`` if the path points to a directory, ``False`` if it points - to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a directory. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a directory. Use :meth:`Path.stat` to distinguish + between these cases. This method normally follows symlinks; to exclude symlinks to directories, add the argument ``follow_symlinks=False``. @@ -1082,11 +1092,10 @@ call fails (for example because the path doesn't exist). .. method:: Path.is_file(*, follow_symlinks=True) - Return ``True`` if the path points to a regular file, ``False`` if it - points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a regular file. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a regular file. Use :meth:`Path.stat` to + distinguish between these cases. This method normally follows symlinks; to exclude symlinks, add the argument ``follow_symlinks=False``. @@ -1122,46 +1131,42 @@ call fails (for example because the path doesn't exist). .. method:: Path.is_symlink() - Return ``True`` if the path points to a symbolic link, ``False`` otherwise. - - ``False`` is also returned if the path doesn't exist; other errors (such - as permission errors) are propagated. + Return ``True`` if the path points to a symbolic link, even if that symlink + is broken. ``False`` will be returned if the path is invalid, inaccessible + or missing, or if it points to something other than a symbolic link. Use + :meth:`Path.stat` to distinguish between these cases. .. method:: Path.is_socket() - Return ``True`` if the path points to a Unix socket (or a symbolic link - pointing to a Unix socket), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a Unix socket. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a Unix socket. Use :meth:`Path.stat` to + distinguish between these cases. .. method:: Path.is_fifo() - Return ``True`` if the path points to a FIFO (or a symbolic link - pointing to a FIFO), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a FIFO. ``False`` will be returned if + the path is invalid, inaccessible or missing, or if it points to something + other than a FIFO. Use :meth:`Path.stat` to distinguish between these + cases. .. method:: Path.is_block_device() - Return ``True`` if the path points to a block device (or a symbolic link - pointing to a block device), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a block device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a block device. Use :meth:`Path.stat` to + distinguish between these cases. .. method:: Path.is_char_device() - Return ``True`` if the path points to a character device (or a symbolic link - pointing to a character device), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a character device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a character device. Use :meth:`Path.stat` to + distinguish between these cases. .. method:: Path.iterdir() diff --git a/Lib/glob.py b/Lib/glob.py index 6088de00a67a99..920f79ad7e1fe5 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -340,7 +340,7 @@ def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): # Low-level methods - lstat = operator.methodcaller('lstat') + lexists = operator.methodcaller('exists', follow_symlinks=False) add_slash = operator.methodcaller('joinpath', '') @staticmethod @@ -516,12 +516,8 @@ def select_exists(self, path, exists=False): # Optimization: this path is already known to exist, e.g. because # it was returned from os.scandir(), so we skip calling lstat(). yield path - else: - try: - self.lstat(path) - yield path - except OSError: - pass + elif self.lexists(path): + yield path @classmethod def walk(cls, root, top_down, on_error, follow_symlinks): @@ -562,7 +558,7 @@ def walk(cls, root, top_down, on_error, follow_symlinks): class _StringGlobber(_Globber): - lstat = staticmethod(os.lstat) + lexists = staticmethod(os.path.lexists) scandir = staticmethod(os.scandir) parse_entry = operator.attrgetter('path') concat_path = operator.add diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 06c10e8e4612ca..568a17df26fc33 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -13,32 +13,12 @@ import functools from glob import _Globber, _no_recurse_symlinks -from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL +from errno import ENOTDIR, ELOOP from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO __all__ = ["UnsupportedOperation"] -# -# Internals -# - -_WINERROR_NOT_READY = 21 # drive exists but is not accessible -_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 -_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself - -# EBADF - guard against macOS `stat` throwing EBADF -_IGNORED_ERRNOS = (ENOENT, ENOTDIR, EBADF, ELOOP) - -_IGNORED_WINERRORS = ( - _WINERROR_NOT_READY, - _WINERROR_INVALID_NAME, - _WINERROR_CANT_RESOLVE_FILENAME) - -def _ignore_error(exception): - return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or - getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) - @functools.cache def _is_case_sensitive(parser): @@ -450,12 +430,7 @@ def exists(self, *, follow_symlinks=True): """ try: self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False return True @@ -465,14 +440,7 @@ def is_dir(self, *, follow_symlinks=True): """ try: return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_file(self, *, follow_symlinks=True): @@ -482,14 +450,7 @@ def is_file(self, *, follow_symlinks=True): """ try: return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_mount(self): @@ -518,13 +479,7 @@ def is_symlink(self): """ try: return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_junction(self): @@ -542,14 +497,7 @@ def is_block_device(self): """ try: return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_char_device(self): @@ -558,14 +506,7 @@ def is_char_device(self): """ try: return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_fifo(self): @@ -574,14 +515,7 @@ def is_fifo(self): """ try: return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def is_socket(self): @@ -590,14 +524,7 @@ def is_socket(self): """ try: return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path + except (OSError, ValueError): return False def samefile(self, other_path): diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index f2c627319d520f..7dc071949b9bd7 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -502,12 +502,46 @@ def stat(self, *, follow_symlinks=True): """ return os.stat(self, follow_symlinks=follow_symlinks) + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + if follow_symlinks: + return os.path.exists(self) + return os.path.lexists(self) + + def is_dir(self, *, follow_symlinks=True): + """ + Whether this path is a directory. + """ + if follow_symlinks: + return os.path.isdir(self) + return PathBase.is_dir(self, follow_symlinks=follow_symlinks) + + def is_file(self, *, follow_symlinks=True): + """ + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + if follow_symlinks: + return os.path.isfile(self) + return PathBase.is_file(self, follow_symlinks=follow_symlinks) + def is_mount(self): """ Check if this path is a mount point """ return os.path.ismount(self) + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + return os.path.islink(self) + def is_junction(self): """ Whether this path is a junction. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst b/Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst new file mode 100644 index 00000000000000..9fad7a416fcc24 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst @@ -0,0 +1,5 @@ +Suppress all :exc:`OSError` exceptions from :meth:`pathlib.Path.exists` and +``is_*()`` methods, rather than a selection of more common errors. The new +behaviour is consistent with :func:`os.path.exists`, :func:`os.path.isdir`, +etc. Use :meth:`Path.stat` to retrieve the file status without suppressing +exceptions. From 7d8725ac6f3304677d71dabdb7c184e98a62d864 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Tue, 14 May 2024 21:14:07 +0100 Subject: [PATCH 074/903] GH-74033: Drop deprecated `pathlib.Path` keyword arguments (#118793) Remove support for supplying keyword arguments to `pathlib.Path()`. This has been deprecated since Python 3.12. --- Doc/whatsnew/3.14.rst | 2 ++ Lib/pathlib/_local.py | 7 ------- Lib/test/test_pathlib/test_pathlib.py | 4 ++-- .../Library/2024-05-08-20-41-48.gh-issue-74033.YebHZj.rst | 1 + 4 files changed, 5 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-08-20-41-48.gh-issue-74033.YebHZj.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 33a0f3e0f2f4bc..27c985bec104fe 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -131,6 +131,8 @@ itertools pathlib ------- +* Remove support for passing additional keyword arguments to + :class:`pathlib.Path`. In previous versions, any such arguments are ignored. * Remove support for passing additional positional arguments to :meth:`pathlib.PurePath.relative_to` and :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 7dc071949b9bd7..011144a565540f 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -483,13 +483,6 @@ class Path(PathBase, PurePath): def _unsupported_msg(cls, attribute): return f"{cls.__name__}.{attribute} is unsupported on this system" - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - def __new__(cls, *args, **kwargs): if cls is Path: cls = WindowsPath if os.name == 'nt' else PosixPath diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 4fd2aac4a62139..3df354eb25a58c 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1108,8 +1108,8 @@ def test_is_mount_root(self): self.assertTrue(R.is_mount()) self.assertFalse((R / '\udfff').is_mount()) - def test_passing_kwargs_deprecated(self): - with self.assertWarns(DeprecationWarning): + def test_passing_kwargs_errors(self): + with self.assertRaises(TypeError): self.cls(foo="bar") def setUpWalk(self): diff --git a/Misc/NEWS.d/next/Library/2024-05-08-20-41-48.gh-issue-74033.YebHZj.rst b/Misc/NEWS.d/next/Library/2024-05-08-20-41-48.gh-issue-74033.YebHZj.rst new file mode 100644 index 00000000000000..e6ff47e1a3e57b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-20-41-48.gh-issue-74033.YebHZj.rst @@ -0,0 +1 @@ +Drop support for passing keyword arguments to :class:`pathlib.Path`. From ee13797dec988884f8792144fe5b3d7f5c8083c9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 14 May 2024 22:39:12 -0400 Subject: [PATCH 075/903] 3.13 What's New: Add PEP 702 (#118922) I honestly forgot this slipped into 3.13, but I think it's worth highlighting more, as it is a PEP-sized change that makes the type system significantly more powerful. @Yhg1s I think it's also worth mentioning in your release announcements. --- Doc/whatsnew/3.13.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e69320e822ab3b..effa554bfe8469 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -104,6 +104,9 @@ New typing features: * :pep:`696`: Type parameters (:data:`typing.TypeVar`, :data:`typing.ParamSpec`, and :data:`typing.TypeVarTuple`) now support defaults. +* :pep:`702`: Support for marking deprecations in the type system using the + new :func:`warnings.deprecated` decorator. + * :pep:`742`: :data:`typing.TypeIs` was added, providing more intuitive type narrowing behavior. From 94591dca510c796c7d40e9b4167ea56f2fdf28ca Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 15 May 2024 11:59:41 +0100 Subject: [PATCH 076/903] gh-118486: Simplify test_win32_mkdir_700 to check the exact ACL (GH-119056) --- Lib/test/test_os.py | 23 ++++++++--------------- Modules/posixmodule.c | 2 +- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 9088318600f4c0..941fa2b2c5c87f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1814,21 +1814,14 @@ def test_exist_ok_existing_regular_file(self): @unittest.skipUnless(os.name == 'nt', "requires Windows") def test_win32_mkdir_700(self): base = os_helper.TESTFN - path1 = os.path.join(os_helper.TESTFN, 'dir1') - path2 = os.path.join(os_helper.TESTFN, 'dir2') - # mode=0o700 is special-cased to override ACLs on Windows - # There's no way to know exactly how the ACLs will look, so we'll - # check that they are different from a regularly created directory. - os.mkdir(path1, mode=0o700) - os.mkdir(path2, mode=0o777) - - out1 = subprocess.check_output(["icacls.exe", path1], encoding="oem") - out2 = subprocess.check_output(["icacls.exe", path2], encoding="oem") - os.rmdir(path1) - os.rmdir(path2) - out1 = out1.replace(path1, "") - out2 = out2.replace(path2, "") - self.assertNotEqual(out1, out2) + path = os.path.abspath(os.path.join(os_helper.TESTFN, 'dir')) + os.mkdir(path, mode=0o700) + out = subprocess.check_output(["cacls.exe", path, "/s"], encoding="oem") + os.rmdir(path) + self.assertEqual( + out.strip(), + f'{path} "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"', + ) def tearDown(self): path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5fe6036b3817e4..710a171580a438 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5590,7 +5590,7 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) if (mode == 0700 /* 0o700 */) { ULONG sdSize; pSecAttr = &secAttr; - // Set a discreationary ACL (D) that is protected (P) and includes + // Set a discretionary ACL (D) that is protected (P) and includes // inheritable (OICI) entries that allow (A) full control (FA) to // SYSTEM (SY), Administrators (BA), and the owner (OW). if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( From 7d722b7d3ac78bfa74a5d2f21513ffbf4f85cff2 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 15 May 2024 17:21:52 +0100 Subject: [PATCH 077/903] Remove references to private symbols from zipimport module docstring (GH-119015) --- Lib/zipimport.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 4e41d109865e85..a49a21f0799df2 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -1,11 +1,9 @@ """zipimport provides support for importing Python modules from Zip archives. -This module exports three objects: +This module exports two objects: - zipimporter: a class; its constructor takes a path to a Zip archive. - ZipImportError: exception raised by zipimporter objects. It's a subclass of ImportError, so it can be caught as ImportError, too. -- _zip_directory_cache: a dict, mapping archive paths to zip directory - info dicts, as used in zipimporter._files. It is usually not needed to use the zipimport module explicitly; it is used by the builtin import mechanism for sys.path items that are paths From 5b88d95cc542cf02303c6fe0e8719a93544decdb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 15 May 2024 19:49:00 +0300 Subject: [PATCH 078/903] gh-118760: Fix errors in calling Tkinter bindings on Windows (GH-118782) For unknown reasons some arguments for Tkinter binding can be created as a 1-tuple containing a Tcl_Obj when wantobjects is 2. --- Lib/tkinter/__init__.py | 3 +++ .../Library/2024-05-08-21-13-56.gh-issue-118760.mdmH3T.rst | 1 + 2 files changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-05-08-21-13-56.gh-issue-118760.mdmH3T.rst diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index daecf4eb2ea522..f03da0ff5f98ec 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1727,6 +1727,9 @@ def getint_event(s): except (ValueError, TclError): return s + if any(isinstance(s, tuple) for s in args): + args = [s[0] if isinstance(s, tuple) and len(s) == 1 else s + for s in args] nsign, b, f, h, k, s, t, w, x, y, A, E, K, N, W, T, X, Y, D = args # Missing: (a, c, d, m, o, v, B, R) e = Event() diff --git a/Misc/NEWS.d/next/Library/2024-05-08-21-13-56.gh-issue-118760.mdmH3T.rst b/Misc/NEWS.d/next/Library/2024-05-08-21-13-56.gh-issue-118760.mdmH3T.rst new file mode 100644 index 00000000000000..89ef9334fbc65d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-21-13-56.gh-issue-118760.mdmH3T.rst @@ -0,0 +1 @@ +Fix errors in calling Tkinter bindings on Windows. From fb0cf7d1408c904e40142a74cd7a53eb52a8e568 Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Wed, 15 May 2024 15:13:52 -0300 Subject: [PATCH 079/903] gh-119009: Add gettext target (#119006) --- Doc/Makefile | 6 ++++++ Doc/conf.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/Doc/Makefile b/Doc/Makefile index dd068c520ad60c..eca574ec290af7 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -32,6 +32,7 @@ help: @echo " clean to remove build files" @echo " venv to create a venv with necessary tools" @echo " html to make standalone HTML files" + @echo " gettext to generate POT files" @echo " htmlview to open the index page built by the html target in your browser" @echo " htmllive to rebuild and reload HTML files in your browser" @echo " htmlhelp to make HTML files and a HTML help project" @@ -140,6 +141,11 @@ pydoc-topics: build @echo "Building finished; now run this:" \ "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" +.PHONY: gettext +gettext: BUILDER = gettext +gettext: SPHINXOPTS += '-d build/doctrees-gettext' +gettext: build + .PHONY: htmlview htmlview: html $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" diff --git a/Doc/conf.py b/Doc/conf.py index 0e86de837d35d2..47fb96fe1de482 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -374,6 +374,8 @@ # Split the index html_split_index = True +# Split pot files one per reST file +gettext_compact = False # Options for LaTeX output # ------------------------ From 4d3ef8056a1b00c002b8a10315c0a00cf646e93b Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Thu, 16 May 2024 07:38:32 +0800 Subject: [PATCH 080/903] Docs: fix typos in documentation (gh-118941) --- Misc/NEWS.d/3.5.0a1.rst | 6 +++--- Misc/NEWS.d/3.6.0a1.rst | 2 +- Misc/NEWS.d/3.6.0b2.rst | 2 +- Misc/NEWS.d/3.6.3rc1.rst | 2 +- Misc/NEWS.d/3.7.0a1.rst | 6 +++--- Misc/NEWS.d/3.7.0a4.rst | 2 +- Misc/NEWS.d/3.7.0b1.rst | 2 +- Misc/NEWS.d/3.7.0b4.rst | 4 ++-- Misc/NEWS.d/3.8.0a1.rst | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Misc/NEWS.d/3.5.0a1.rst b/Misc/NEWS.d/3.5.0a1.rst index 5244db107a73da..442ab62fee8185 100644 --- a/Misc/NEWS.d/3.5.0a1.rst +++ b/Misc/NEWS.d/3.5.0a1.rst @@ -1345,7 +1345,7 @@ newer worked. .. section: Library The "ip" command is now used on Linux to determine MAC address in -uuid.getnode(). Pach by Bruno Cauet. +uuid.getnode(). Patch by Bruno Cauet. .. @@ -3930,7 +3930,7 @@ has been called. .. nonce: 5CDoox .. section: Library -New keyword argument ``unsafe`` to Mock. It raises ``AttributeError`` incase of +New keyword argument ``unsafe`` to Mock. It raises ``AttributeError`` in case of an attribute startswith assert or assret. .. @@ -4339,7 +4339,7 @@ these modules are not used. .. nonce: V1-XhC .. section: Library -Include the broadcast address in the usuable hosts for IPv6 in ipaddress. +Include the broadcast address in the usable hosts for IPv6 in ipaddress. .. diff --git a/Misc/NEWS.d/3.6.0a1.rst b/Misc/NEWS.d/3.6.0a1.rst index 144d217f6098a1..5c9a6e5d64b469 100644 --- a/Misc/NEWS.d/3.6.0a1.rst +++ b/Misc/NEWS.d/3.6.0a1.rst @@ -1401,7 +1401,7 @@ array is extended. .. section: Library doctest.DocFileTest and doctest.testfile() now support packages (module -splitted into multiple directories) for the package parameter. +split into multiple directories) for the package parameter. .. diff --git a/Misc/NEWS.d/3.6.0b2.rst b/Misc/NEWS.d/3.6.0b2.rst index 9413c6e01917d5..23dd69efb23b88 100644 --- a/Misc/NEWS.d/3.6.0b2.rst +++ b/Misc/NEWS.d/3.6.0b2.rst @@ -215,7 +215,7 @@ memcpy(). .. nonce: e5xc1i .. section: Core and Builtins -Fix dict.pop() for splitted dictionary when trying to remove a "pending key" +Fix dict.pop() for split dictionary when trying to remove a "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. .. diff --git a/Misc/NEWS.d/3.6.3rc1.rst b/Misc/NEWS.d/3.6.3rc1.rst index ebda7665e2b6ea..6a20e07f05956c 100644 --- a/Misc/NEWS.d/3.6.3rc1.rst +++ b/Misc/NEWS.d/3.6.3rc1.rst @@ -85,7 +85,7 @@ wrong line (typically the first line of the file). .. nonce: Kl_fS5 .. section: Core and Builtins -Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C libray +Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C library plans to remove the functions from sys/types.h. .. diff --git a/Misc/NEWS.d/3.7.0a1.rst b/Misc/NEWS.d/3.7.0a1.rst index 58d51c420a10ae..fd6ba07b53a617 100644 --- a/Misc/NEWS.d/3.7.0a1.rst +++ b/Misc/NEWS.d/3.7.0a1.rst @@ -214,7 +214,7 @@ Fix possible undefined behavior in _PyObject_FastCall_Prepend. .. nonce: Kl_fS5 .. section: Core and Builtins -Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C libray +Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C library plans to remove the functions from sys/types.h. .. @@ -1479,7 +1479,7 @@ memcpy(). .. nonce: e5xc1i .. section: Core and Builtins -Fix dict.pop() for splitted dictionary when trying to remove a "pending key" +Fix dict.pop() for split dictionary when trying to remove a "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. .. @@ -2516,7 +2516,7 @@ stdin.write() if the child process is still running but closed the pipe. .. nonce: CdOuSl .. section: Library -Addded empty __slots__ to abc.ABC. This allows subclassers to deny __dict__ +Added empty __slots__ to abc.ABC. This allows subclassers to deny __dict__ and __weakref__ creation. Patch by Aaron Hall. .. diff --git a/Misc/NEWS.d/3.7.0a4.rst b/Misc/NEWS.d/3.7.0a4.rst index f2c6559037d84f..679f72ee0a44d4 100644 --- a/Misc/NEWS.d/3.7.0a4.rst +++ b/Misc/NEWS.d/3.7.0a4.rst @@ -595,7 +595,7 @@ Add asyncio.get_running_loop() function. .. section: Library All class and static methods of builtin types now are correctly classified -by inspect.classify_class_attrs() and grouped in pydoc ouput. Added +by inspect.classify_class_attrs() and grouped in pydoc output. Added types.ClassMethodDescriptorType for unbound class methods of builtin types. .. diff --git a/Misc/NEWS.d/3.7.0b1.rst b/Misc/NEWS.d/3.7.0b1.rst index d1beec9cdcc33a..b6477127818eb5 100644 --- a/Misc/NEWS.d/3.7.0b1.rst +++ b/Misc/NEWS.d/3.7.0b1.rst @@ -875,4 +875,4 @@ by Stéphane Wirtel .. section: C API Add C API access to the ``datetime.timezone`` constructor and -``datetime.timzone.UTC`` singleton. +``datetime.timezone.UTC`` singleton. diff --git a/Misc/NEWS.d/3.7.0b4.rst b/Misc/NEWS.d/3.7.0b4.rst index b17c7e08d1d408..fd0ce25cd8fb41 100644 --- a/Misc/NEWS.d/3.7.0b4.rst +++ b/Misc/NEWS.d/3.7.0b4.rst @@ -46,8 +46,8 @@ Fix potential memory leak in ``normalizestring()``. Change dict growth function from ``round_up_to_power_2(used*2+hashtable_size/2)`` to -``round_up_to_power_2(used*3)``. Previously, dict is shrinked only when -``used == 0``. Now dict has more chance to be shrinked. +``round_up_to_power_2(used*3)``. Previously, dict is shrunk only when +``used == 0``. Now dict has more chance to be shrunk. .. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 1964a8329979f5..258e7d82b3f303 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -1406,8 +1406,8 @@ Fix potential memory leak in ``normalizestring()``. Change dict growth function from ``round_up_to_power_2(used*2+hashtable_size/2)`` to -``round_up_to_power_2(used*3)``. Previously, dict is shrinked only when -``used == 0``. Now dict has more chance to be shrinked. +``round_up_to_power_2(used*3)``. Previously, dict is shrunk only when +``used == 0``. Now dict has more chance to be shrunk. .. From 66b73e9724fc376715ae264c8282dc1e981e4f17 Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Thu, 16 May 2024 02:13:47 -0300 Subject: [PATCH 081/903] Use literal syntax in origin property (#119029) --- Doc/library/importlib.metadata.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 674ce5807fdf11..9c0879f5ca850f 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -343,7 +343,7 @@ instance:: >>> dist.metadata['License'] # doctest: +SKIP 'MIT' -For editable packages, an origin property may present :pep:`610` +For editable packages, an ``origin`` property may present :pep:`610` metadata:: >>> dist.origin.url From 0142a2292c3d3bfa56a987d576a9678be0f56931 Mon Sep 17 00:00:00 2001 From: Wulian233 <71213467+Wulian233@users.noreply.github.com> Date: Thu, 16 May 2024 13:16:34 +0800 Subject: [PATCH 082/903] Fix typos in test_buffer.py and update numpy issue links (#118963) --- Lib/test/test_buffer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 5b1b95b9c82064..ae938d12c9401b 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -130,10 +130,10 @@ def native_type_range(fmt): for fmt in fmtdict['@']: fmtdict['@'][fmt] = native_type_range(fmt) -# Format codes suppported by the memoryview object +# Format codes supported by the memoryview object MEMORYVIEW = NATIVE.copy() -# Format codes suppported by array.array +# Format codes supported by array.array ARRAY = NATIVE.copy() for k in NATIVE: if not k in "bBhHiIlLfd": @@ -168,7 +168,7 @@ def randrange_fmt(mode, char, obj): if char == 'c': x = bytes([x]) if obj == 'numpy' and x == b'\x00': - # http://projects.scipy.org/numpy/ticket/1925 + # https://github.com/numpy/numpy/issues/2518 x = b'\x01' if char == '?': x = bool(x) @@ -1918,7 +1918,7 @@ def test_ndarray_random(self): if numpy_array: shape = t[3] if 0 in shape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 z = numpy_array_from_structure(items, fmt, t) self.verify(x, obj=None, itemsize=z.itemsize, fmt=fmt, readonly=False, @@ -1950,7 +1950,7 @@ def test_ndarray_random_invalid(self): except Exception as e: numpy_err = e.__class__ - if 0: # http://projects.scipy.org/numpy/ticket/1910 + if 0: # https://github.com/numpy/numpy/issues/2503 self.assertTrue(numpy_err) def test_ndarray_random_slice_assign(self): @@ -1996,7 +1996,7 @@ def test_ndarray_random_slice_assign(self): if numpy_array: if 0 in lshape or 0 in rshape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 zl = numpy_array_from_structure(litems, fmt, tl) zr = numpy_array_from_structure(ritems, fmt, tr) From 0152dc4ff5534fa2948b95262e70ff6b202b9b99 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 16 May 2024 10:25:10 +0300 Subject: [PATCH 083/903] gh-119064: Use os_helper.FakePath instead of pathlib.Path in tests (GH-119065) --- Lib/test/_test_multiprocessing.py | 6 +-- Lib/test/test_asyncio/test_unix_events.py | 11 ++--- Lib/test/test_compileall.py | 26 +++++------ Lib/test/test_configparser.py | 5 +- Lib/test/test_ctypes/test_loading.py | 5 +- Lib/test/test_fileinput.py | 17 ++++--- Lib/test/test_http_cookiejar.py | 7 ++- Lib/test/test_logging.py | 6 +-- Lib/test/test_mimetypes.py | 13 ++++-- Lib/test/test_pkgutil.py | 3 +- Lib/test/test_runpy.py | 13 +++--- Lib/test/test_shutil.py | 8 ++-- Lib/test/test_subprocess.py | 7 +-- Lib/test/test_tarfile.py | 56 ++++++++++++----------- Lib/test/test_tempfile.py | 16 ++----- Lib/test/test_venv.py | 10 ++-- Lib/test/test_winsound.py | 14 ++---- Lib/test/test_zipapp.py | 9 ++-- Lib/test/test_zipfile/_path/test_path.py | 10 ++-- 19 files changed, 115 insertions(+), 127 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 46afdfca331a23..f126b6745dc83b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -22,7 +22,6 @@ import subprocess import struct import operator -import pathlib import pickle import weakref import warnings @@ -324,8 +323,9 @@ def test_set_executable(self): self.skipTest(f'test not appropriate for {self.TYPE}') paths = [ sys.executable, # str - sys.executable.encode(), # bytes - pathlib.Path(sys.executable) # os.PathLike + os.fsencode(sys.executable), # bytes + os_helper.FakePath(sys.executable), # os.PathLike + os_helper.FakePath(os.fsencode(sys.executable)), # os.PathLike bytes ] for path in paths: self.set_executable(path) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 59ef9f5f58cabc..9452213c685851 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -6,7 +6,6 @@ import multiprocessing from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests import os -import pathlib import signal import socket import stat @@ -304,20 +303,20 @@ def test_create_unix_server_existing_path_sock(self): self.loop.run_until_complete(srv.wait_closed()) @socket_helper.skip_unless_bind_unix_socket - def test_create_unix_server_pathlib(self): + def test_create_unix_server_pathlike(self): with test_utils.unix_socket_path() as path: - path = pathlib.Path(path) + path = os_helper.FakePath(path) srv_coro = self.loop.create_unix_server(lambda: None, path) srv = self.loop.run_until_complete(srv_coro) srv.close() self.loop.run_until_complete(srv.wait_closed()) - def test_create_unix_connection_pathlib(self): + def test_create_unix_connection_pathlike(self): with test_utils.unix_socket_path() as path: - path = pathlib.Path(path) + path = os_helper.FakePath(path) coro = self.loop.create_unix_connection(lambda: None, path) with self.assertRaises(FileNotFoundError): - # If pathlib.Path wasn't supported, the exception would be + # If path-like object weren't supported, the exception would be # different. self.loop.run_until_complete(coro) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index bf0fd051672db4..812ff5e7f84461 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -4,7 +4,6 @@ import importlib.util import io import os -import pathlib import py_compile import shutil import struct @@ -31,6 +30,7 @@ from test.support import script_helper from test.test_py_compile import without_source_date_epoch from test.test_py_compile import SourceDateEpochTestMeta +from test.support.os_helper import FakePath def get_pyc(script, opt): @@ -156,28 +156,28 @@ def test_compile_file_pathlike(self): self.assertFalse(os.path.isfile(self.bc_path)) # we should also test the output with support.captured_stdout() as stdout: - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path))) + self.assertTrue(compileall.compile_file(FakePath(self.source_path))) self.assertRegex(stdout.getvalue(), r'Compiling ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_ddir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - ddir=pathlib.Path('ddir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + ddir=FakePath('ddir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_stripdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - stripdir=pathlib.Path('stripdir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + stripdir=FakePath('stripdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_prependdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - prependdir=pathlib.Path('prependdir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + prependdir=FakePath('prependdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) @@ -228,22 +228,22 @@ def test_optimize(self): def test_compile_dir_pathlike(self): self.assertFalse(os.path.isfile(self.bc_path)) with support.captured_stdout() as stdout: - compileall.compile_dir(pathlib.Path(self.directory)) + compileall.compile_dir(FakePath(self.directory)) line = stdout.getvalue().splitlines()[0] self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_dir_pathlike_stripdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), - stripdir=pathlib.Path('stripdir_path'), + self.assertTrue(compileall.compile_dir(FakePath(self.directory), + stripdir=FakePath('stripdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_dir_pathlike_prependdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), - prependdir=pathlib.Path('prependdir_path'), + self.assertTrue(compileall.compile_dir(FakePath(self.directory), + prependdir=FakePath('prependdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index fe09472db89cd2..a934e493a76391 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,7 +2,6 @@ import configparser import io import os -import pathlib import textwrap import unittest @@ -745,12 +744,12 @@ def test_read_returns_file_list(self): self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only a Path object: cf = self.newconfig() - parsed_files = cf.read(pathlib.Path(file1), encoding="utf-8") + parsed_files = cf.read(os_helper.FakePath(file1), encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we passed both a filename and a Path object: cf = self.newconfig() - parsed_files = cf.read([pathlib.Path(file1), file1], encoding="utf-8") + parsed_files = cf.read([os_helper.FakePath(file1), file1], encoding="utf-8") self.assertEqual(parsed_files, [file1, file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index b25e81b65cf103..fc1eecb77e17e3 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -42,10 +42,7 @@ def test_load(self): self.skipTest('could not find library to load') CDLL(test_lib) CDLL(os.path.basename(test_lib)) - class CTypesTestPathLikeCls: - def __fspath__(self): - return test_lib - CDLL(CTypesTestPathLikeCls()) + CDLL(os_helper.FakePath(test_lib)) self.assertRaises(OSError, CDLL, self.unknowndll) def test_load_version(self): diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index b3ad41d2588c4c..b340ef7ed1621c 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -23,10 +23,9 @@ from io import BytesIO, StringIO from fileinput import FileInput, hook_encoded -from pathlib import Path from test.support import verbose -from test.support.os_helper import TESTFN +from test.support.os_helper import TESTFN, FakePath from test.support.os_helper import unlink as safe_unlink from test.support import os_helper from test import support @@ -478,23 +477,23 @@ def test_iteration_buffering(self): self.assertRaises(StopIteration, next, fi) self.assertEqual(src.linesread, []) - def test_pathlib_file(self): - t1 = Path(self.writeTmp("Pathlib file.")) + def test_pathlike_file(self): + t1 = FakePath(self.writeTmp("Path-like file.")) with FileInput(t1, encoding="utf-8") as fi: line = fi.readline() - self.assertEqual(line, 'Pathlib file.') + self.assertEqual(line, 'Path-like file.') self.assertEqual(fi.lineno(), 1) self.assertEqual(fi.filelineno(), 1) self.assertEqual(fi.filename(), os.fspath(t1)) - def test_pathlib_file_inplace(self): - t1 = Path(self.writeTmp('Pathlib file.')) + def test_pathlike_file_inplace(self): + t1 = FakePath(self.writeTmp('Path-like file.')) with FileInput(t1, inplace=True, encoding="utf-8") as fi: line = fi.readline() - self.assertEqual(line, 'Pathlib file.') + self.assertEqual(line, 'Path-like file.') print('Modified %s' % line) with open(t1, encoding="utf-8") as f: - self.assertEqual(f.read(), 'Modified Pathlib file.\n') + self.assertEqual(f.read(), 'Modified Path-like file.\n') class MockFileInput: diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index 97e9c82cde9ec4..dbf9ce10f76f91 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -9,7 +9,6 @@ import time import unittest import urllib.request -import pathlib from http.cookiejar import (time2isoz, http2time, iso2time, time2netscape, parse_ns_headers, join_header_words, split_header_words, Cookie, @@ -337,9 +336,9 @@ def test_constructor_with_str(self): self.assertEqual(c.filename, filename) def test_constructor_with_path_like(self): - filename = pathlib.Path(os_helper.TESTFN) - c = LWPCookieJar(filename) - self.assertEqual(c.filename, os.fspath(filename)) + filename = os_helper.TESTFN + c = LWPCookieJar(os_helper.FakePath(filename)) + self.assertEqual(c.filename, filename) def test_constructor_with_none(self): c = LWPCookieJar(None) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index e651d96f100a83..97d7c9fb167ec1 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -657,15 +657,15 @@ def test_builtin_handlers(self): self.assertFalse(h.shouldFlush(r)) h.close() - def test_path_objects(self): + def test_pathlike_objects(self): """ - Test that Path objects are accepted as filename arguments to handlers. + Test that path-like objects are accepted as filename arguments to handlers. See Issue #27493. """ fn = make_temp_file() os.unlink(fn) - pfn = pathlib.Path(fn) + pfn = os_helper.FakePath(fn) cases = ( (logging.FileHandler, (pfn, 'w')), (logging.handlers.RotatingFileHandler, (pfn, 'a')), diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 2e0ad0606ae9c2..58f6a4dfae08ba 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -1,7 +1,6 @@ import io import mimetypes import os -import pathlib import sys import unittest.mock @@ -83,11 +82,19 @@ def test_read_mime_types(self): with os_helper.temp_dir() as directory: data = "x-application/x-unittest pyunit\n" - file = pathlib.Path(directory, "sample.mimetype") - file.write_text(data, encoding="utf-8") + file = os.path.join(directory, "sample.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) mime_dict = mimetypes.read_mime_types(file) eq(mime_dict[".pyunit"], "x-application/x-unittest") + data = "x-application/x-unittest2 pyunit2\n" + file = os.path.join(directory, "sample2.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) + mime_dict = mimetypes.read_mime_types(os_helper.FakePath(file)) + eq(mime_dict[".pyunit2"], "x-application/x-unittest2") + # bpo-41048: read_mime_types should read the rule file with 'utf-8' encoding. # Not with locale encoding. _bootlocale has been imported because io.open(...) # uses it. diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index e19dce1dbd2583..d095f440a99f63 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -13,6 +13,7 @@ import zipfile from test.support.import_helper import DirsOnSysPath +from test.support.os_helper import FakePath from test.test_importlib.util import uncache # Note: pkgutil.walk_packages is currently tested in test_runpy. This is @@ -121,7 +122,7 @@ def test_issue44061_iter_modules(self): # make sure iter_modules accepts Path objects names = [] - for moduleinfo in pkgutil.iter_modules([Path(zip_file)]): + for moduleinfo in pkgutil.iter_modules([FakePath(zip_file)]): self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo) names.append(moduleinfo.name) self.assertEqual(names, [pkg]) diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 9d76764c75be3e..b64383f6546f31 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -15,7 +15,7 @@ from test.support import (infinite_recursion, no_tracing, verbose, requires_subprocess, requires_resource) from test.support.import_helper import forget, make_legacy_pyc, unload -from test.support.os_helper import create_empty_file, temp_dir +from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script @@ -657,14 +657,13 @@ def test_basic_script(self): self._check_script(script_name, "", script_name, script_name, expect_spec=False) - def test_basic_script_with_path_object(self): + def test_basic_script_with_pathlike_object(self): with temp_dir() as script_dir: mod_name = 'script' - script_name = pathlib.Path(self._make_test_script(script_dir, - mod_name)) - self._check_script(script_name, "", - os.fsdecode(script_name), - os.fsdecode(script_name), + script_name = self._make_test_script(script_dir, mod_name) + self._check_script(FakePath(script_name), "", + script_name, + script_name, expect_spec=False) def test_basic_script_no_suffix(self): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 5b0aac67a0adeb..df9e7a660bf29e 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -914,7 +914,7 @@ def _ignore(src, names): 'test.txt'))) dst_dir = join(self.mkdtemp(), 'destination') - shutil.copytree(pathlib.Path(src_dir), dst_dir, ignore=_ignore) + shutil.copytree(FakePath(src_dir), dst_dir, ignore=_ignore) self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) @@ -2107,7 +2107,7 @@ def check_unpack_archive(self, format, **kwargs): self.check_unpack_archive_with_converter( format, lambda path: path, **kwargs) self.check_unpack_archive_with_converter( - format, pathlib.Path, **kwargs) + format, FakePath, **kwargs) self.check_unpack_archive_with_converter(format, FakePath, **kwargs) def check_unpack_archive_with_converter(self, format, converter, **kwargs): @@ -2672,12 +2672,12 @@ def test_move_file_to_dir(self): def test_move_file_to_dir_pathlike_src(self): # Move a pathlike file to another location on the same filesystem. - src = pathlib.Path(self.src_file) + src = FakePath(self.src_file) self._check_move_file(src, self.dst_dir, self.dst_file) def test_move_file_to_dir_pathlike_dst(self): # Move a file to another pathlike location on the same filesystem. - dst = pathlib.Path(self.dst_dir) + dst = FakePath(self.dst_dir) self._check_move_file(self.src_file, dst, self.dst_file) @mock_rename diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 9ecd8426cb5537..8b69cd03ba7f24 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -25,7 +25,6 @@ import gc import textwrap import json -import pathlib from test.support.os_helper import FakePath try: @@ -1522,9 +1521,6 @@ def test_communicate_epipe(self): p.communicate(b"x" * 2**20) def test_repr(self): - path_cmd = pathlib.Path("my-tool.py") - pathlib_cls = path_cmd.__class__.__name__ - cases = [ ("ls", True, 123, ""), ('a' * 100, True, 0, @@ -1532,7 +1528,8 @@ def test_repr(self): (["ls"], False, None, ""), (["ls", '--my-opts', 'a' * 100], False, None, ""), - (path_cmd, False, 7, f"") + (os_helper.FakePath("my-tool.py"), False, 7, + ">") ] with unittest.mock.patch.object(subprocess.Popen, '_execute_child'): for cmd, shell, code, sx in cases: diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index c9c1097963a885..f715940de1d584 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -386,7 +386,7 @@ def test_is_tarfile_erroneous(self): self.assertFalse(tarfile.is_tarfile(tmpname)) # is_tarfile works on path-like objects - self.assertFalse(tarfile.is_tarfile(pathlib.Path(tmpname))) + self.assertFalse(tarfile.is_tarfile(os_helper.FakePath(tmpname))) # is_tarfile works on file objects with open(tmpname, "rb") as fobj: @@ -400,7 +400,7 @@ def test_is_tarfile_valid(self): self.assertTrue(tarfile.is_tarfile(self.tarname)) # is_tarfile works on path-like objects - self.assertTrue(tarfile.is_tarfile(pathlib.Path(self.tarname))) + self.assertTrue(tarfile.is_tarfile(os_helper.FakePath(self.tarname))) # is_tarfile works on file objects with open(self.tarname, "rb") as fobj: @@ -576,21 +576,23 @@ def test_bytes_name_attribute(self): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) - def test_pathlike_name(self): - tarname = pathlib.Path(self.tarname) + def test_pathlike_name(self, tarname=None): + if tarname is None: + tarname = self.tarname + expected = os.path.abspath(tarname) + tarname = os_helper.FakePath(tarname) with tarfile.open(tarname, mode=self.mode) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) with self.taropen(tarname) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) with tarfile.TarFile.open(tarname, mode=self.mode) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) if self.suffix == '': with tarfile.TarFile(tarname, mode='r') as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) + + def test_pathlike_bytes_name(self): + self.test_pathlike_name(os.fsencode(self.tarname)) def test_illegal_mode_arg(self): with open(tmpname, 'wb'): @@ -761,24 +763,24 @@ def test_deprecation_if_no_filter_passed_to_extract(self): # check that the stacklevel of the deprecation warning is correct: self.assertEqual(cm.filename, __file__) - def test_extractall_pathlike_name(self): - DIR = pathlib.Path(TEMPDIR) / "extractall" + def test_extractall_pathlike_dir(self): + DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: directories = [t for t in tar if t.isdir()] - tar.extractall(DIR, directories, filter='fully_trusted') + tar.extractall(os_helper.FakePath(DIR), directories, filter='fully_trusted') for tarinfo in directories: - path = DIR / tarinfo.name + path = os.path.join(DIR, tarinfo.name) self.assertEqual(os.path.getmtime(path), tarinfo.mtime) - def test_extract_pathlike_name(self): + def test_extract_pathlike_dir(self): dirtype = "ustar/dirtype" - DIR = pathlib.Path(TEMPDIR) / "extractall" + DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: tarinfo = tar.getmember(dirtype) - tar.extract(tarinfo, path=DIR, filter='fully_trusted') - extracted = DIR / dirtype + tar.extract(tarinfo, path=os_helper.FakePath(DIR), filter='fully_trusted') + extracted = os.path.join(DIR, dirtype) self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) def test_init_close_fobj(self): @@ -1390,11 +1392,11 @@ def test_ordered_recursion(self): def test_gettarinfo_pathlike_name(self): with tarfile.open(tmpname, self.mode) as tar: - path = pathlib.Path(TEMPDIR) / "file" + path = os.path.join(TEMPDIR, "file") with open(path, "wb") as fobj: fobj.write(b"aaa") - tarinfo = tar.gettarinfo(path) - tarinfo2 = tar.gettarinfo(os.fspath(path)) + tarinfo = tar.gettarinfo(os_helper.FakePath(path)) + tarinfo2 = tar.gettarinfo(path) self.assertIsInstance(tarinfo.name, str) self.assertEqual(tarinfo.name, tarinfo2.name) self.assertEqual(tarinfo.size, 3) @@ -1947,10 +1949,10 @@ def test_create_existing_taropen(self): self.assertIn("spameggs42", names[0]) def test_create_pathlike_name(self): - with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + with tarfile.open(os_helper.FakePath(tmpname), self.mode) as tobj: self.assertIsInstance(tobj.name, str) self.assertEqual(tobj.name, os.path.abspath(tmpname)) - tobj.add(pathlib.Path(self.file_path)) + tobj.add(os_helper.FakePath(self.file_path)) names = tobj.getnames() self.assertEqual(len(names), 1) self.assertIn('spameggs42', names[0]) @@ -1961,10 +1963,10 @@ def test_create_pathlike_name(self): self.assertIn('spameggs42', names[0]) def test_create_taropen_pathlike_name(self): - with self.taropen(pathlib.Path(tmpname), "x") as tobj: + with self.taropen(os_helper.FakePath(tmpname), "x") as tobj: self.assertIsInstance(tobj.name, str) self.assertEqual(tobj.name, os.path.abspath(tmpname)) - tobj.add(pathlib.Path(self.file_path)) + tobj.add(os_helper.FakePath(self.file_path)) names = tobj.getnames() self.assertEqual(len(names), 1) self.assertIn('spameggs42', names[0]) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 19ddeaa169bf93..a5e182cef23dc5 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -63,16 +63,10 @@ def test_infer_return_type_multiples_and_none(self): tempfile._infer_return_type(b'', None, '') def test_infer_return_type_pathlib(self): - self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/'))) + self.assertIs(str, tempfile._infer_return_type(os_helper.FakePath('/'))) def test_infer_return_type_pathlike(self): - class Path: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - + Path = os_helper.FakePath self.assertIs(str, tempfile._infer_return_type(Path('/'))) self.assertIs(bytes, tempfile._infer_return_type(Path(b'/'))) self.assertIs(str, tempfile._infer_return_type('', Path(''))) @@ -443,7 +437,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir).write(b"blat") - self.do_create(dir=pathlib.Path(dir)).write(b"blat") + self.do_create(dir=os_helper.FakePath(dir)).write(b"blat") finally: support.gc_collect() # For PyPy or other GCs. os.rmdir(dir) @@ -681,7 +675,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir) - self.do_create(dir=pathlib.Path(dir)) + self.do_create(dir=os_helper.FakePath(dir)) finally: os.rmdir(dir) @@ -782,7 +776,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: os.rmdir(self.do_create(dir=dir)) - os.rmdir(self.do_create(dir=pathlib.Path(dir))) + os.rmdir(self.do_create(dir=os_helper.FakePath(dir))) finally: os.rmdir(dir) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 668642f73e8d9f..1769ed61b94075 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -24,7 +24,7 @@ requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, - TESTFN) + TESTFN, FakePath) import unittest import venv from unittest.mock import patch, Mock @@ -125,12 +125,12 @@ def test_defaults_with_str_path(self): self.run_with_capture(venv.create, self.env_dir) self._check_output_of_default_create() - def test_defaults_with_pathlib_path(self): + def test_defaults_with_pathlike(self): """ - Test the create function with default arguments and a pathlib.Path path. + Test the create function with default arguments and a path-like path. """ rmtree(self.env_dir) - self.run_with_capture(venv.create, pathlib.Path(self.env_dir)) + self.run_with_capture(venv.create, FakePath(self.env_dir)) self._check_output_of_default_create() def _check_output_of_default_create(self): @@ -572,7 +572,7 @@ def test_pathsep_error(self): rmtree(self.env_dir) bad_itempath = self.env_dir + os.pathsep self.assertRaises(ValueError, venv.create, bad_itempath) - self.assertRaises(ValueError, venv.create, pathlib.Path(bad_itempath)) + self.assertRaises(ValueError, venv.create, FakePath(bad_itempath)) @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') @requireVenvCreate diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py index a59d0d24f5db48..870ab7bd41d8ce 100644 --- a/Lib/test/test_winsound.py +++ b/Lib/test/test_winsound.py @@ -1,12 +1,13 @@ # Ridiculously simple test of the winsound module for Windows. import functools -import pathlib +import os import time import unittest from test import support from test.support import import_helper +from test.support import os_helper support.requires('audio') @@ -85,13 +86,6 @@ def test_keyword_args(self): safe_MessageBeep(type=winsound.MB_OK) -# A class for testing winsound when the given path resolves -# to bytes rather than str. -class BytesPath(pathlib.WindowsPath): - def __fspath__(self): - return bytes(super().__fspath__(), 'UTF-8') - - class PlaySoundTest(unittest.TestCase): def test_errors(self): @@ -126,7 +120,7 @@ def test_snd_filename(self): def test_snd_filepath(self): fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') - path = pathlib.Path(fn) + path = os_helper.FakePath(fn) safe_PlaySound(path, winsound.SND_FILENAME | winsound.SND_NODEFAULT) def test_snd_filepath_as_bytes(self): @@ -134,7 +128,7 @@ def test_snd_filepath_as_bytes(self): self.assertRaises( TypeError, winsound.PlaySound, - BytesPath(fn), + os_helper.FakePath(os.fsencode(fn)), winsound.SND_FILENAME | winsound.SND_NODEFAULT ) diff --git a/Lib/test/test_zipapp.py b/Lib/test/test_zipapp.py index f1c6b2d97621ee..00a5ed6626ddc5 100644 --- a/Lib/test/test_zipapp.py +++ b/Lib/test/test_zipapp.py @@ -265,14 +265,15 @@ def test_write_shebang_to_fileobj(self): zipapp.create_archive(str(target), new_target, interpreter='python2.7') self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) - def test_read_from_pathobj(self): - # Test that we can copy an archive using a pathlib.Path object + def test_read_from_pathlike_obj(self): + # Test that we can copy an archive using a path-like object # for the source. source = self.tmpdir / 'source' source.mkdir() (source / '__main__.py').touch() - target1 = self.tmpdir / 'target1.pyz' - target2 = self.tmpdir / 'target2.pyz' + source = os_helper.FakePath(str(source)) + target1 = os_helper.FakePath(str(self.tmpdir / 'target1.pyz')) + target2 = os_helper.FakePath(str(self.tmpdir / 'target2.pyz')) zipapp.create_archive(source, target1, interpreter='python') zipapp.create_archive(target1, target2, interpreter='python2.7') self.assertEqual(zipapp.get_interpreter(target2), 'python2.7') diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index df5b8c9d8fea40..f6e2c8c289f6fd 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -13,7 +13,7 @@ from ._test_params import parameterize, Invoked -from test.support.os_helper import temp_dir +from test.support.os_helper import temp_dir, FakePath class jaraco: @@ -264,13 +264,13 @@ def test_pathlike_construction(self, alpharep): zipfile.Path should be constructable from a path-like object """ zipfile_ondisk = self.zipfile_ondisk(alpharep) - pathlike = pathlib.Path(str(zipfile_ondisk)) + pathlike = FakePath(str(zipfile_ondisk)) zipfile.Path(pathlike) @pass_alpharep def test_traverse_pathlike(self, alpharep): root = zipfile.Path(alpharep) - root / pathlib.Path("a") + root / FakePath("a") @pass_alpharep def test_parent(self, alpharep): @@ -539,12 +539,12 @@ def test_inheritance(self, alpharep): ['alpharep', 'path_type', 'subpath'], itertools.product( alpharep_generators, - [str, pathlib.Path], + [str, FakePath], ['', 'b/'], ), ) def test_pickle(self, alpharep, path_type, subpath): - zipfile_ondisk = path_type(self.zipfile_ondisk(alpharep)) + zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) restored_1 = pickle.loads(saved_1) From b6839942a8906fccdd64e749abeefe8a61ce7e03 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 16 May 2024 11:16:46 +0300 Subject: [PATCH 084/903] Add Tkinter tests for different events (GH-118778) --- Lib/test/test_tkinter/test_misc.py | 278 +++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 6dca2a3920e06a..d9ea642881a179 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -532,6 +532,284 @@ def test_wm_attribute(self): 1.0 if self.wantobjects else '1.0') +class EventTest(AbstractTkTest, unittest.TestCase): + + def test_focus(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.focus_force() + self.root.update() + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.FocusIn) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, '??') + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, '??') + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, '??') + self.assertEqual(e.y, '??') + self.assertEqual(e.x_root, '??') + self.assertEqual(e.y_root, '??') + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_configure(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.configure(height=120, borderwidth=10) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Configure) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, '??') + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, '??') + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, 150) + self.assertEqual(e.height, 100) + self.assertEqual(e.x, 0) + self.assertEqual(e.y, 0) + self.assertEqual(e.x_root, '??') + self.assertEqual(e.y_root, '??') + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_key_press(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('') + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.KeyPress) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertIsInstance(e.state, int) + self.assertNotEqual(e.state, 0) + self.assertEqual(e.char, 'z') + self.assertIsInstance(e.keycode, int) + self.assertNotEqual(e.keycode, 0) + self.assertEqual(e.keysym, 'z') + self.assertEqual(e.keysym_num, ord('z')) + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, -1 - f.winfo_rootx()) + self.assertEqual(e.y, -1 - f.winfo_rooty()) + self.assertEqual(e.x_root, -1) + self.assertEqual(e.y_root, -1) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), + f"") + + def test_event_generate_enter(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Enter) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertIs(e.focus, False) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, 100 + f.winfo_rootx()) + self.assertEqual(e.y_root, 50 + f.winfo_rooty()) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_button_press(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.ButtonPress) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, 1) + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_motion(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Motion) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0x100) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_mouse_wheel(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50, delta=-5) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.MouseWheel) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.time, 0) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, -5) + self.assertEqual(repr(e), '') + + def test_generate_event_virtual_event(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('<>', events.append) + f.focus_force() + + f.event_generate('<>', x=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.VirtualEvent) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 50) + self.assertEqual(e.y, 0) + self.assertEqual(e.x_root, f.winfo_rootx() + 50) + self.assertEqual(e.y_root, -1) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), + f"") + + class BindTest(AbstractTkTest, unittest.TestCase): def setUp(self): From 17cba55786a1b1e6b715b1a88ae1f9088f5d5999 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 16 May 2024 06:39:37 -0700 Subject: [PATCH 085/903] gh-108267: Fix object.__setattr__ regression in dataclasses docs (#119082) --- Doc/library/dataclasses.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 98459c1b6e1020..7aa754c9ccc0a1 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -615,7 +615,8 @@ methods will raise a :exc:`FrozenInstanceError` when invoked. There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and -must use :meth:`!__setattr__`. +must use :meth:`!object.__setattr__`. +.. Make sure to not remove "object" from "object.__setattr__" in the above markup .. _dataclasses-inheritance: From ab73bcdf73fed5a23f2e2e37a63d6992f29479a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 16 May 2024 18:09:52 +0200 Subject: [PATCH 086/903] Explain how to install LLVM on Fedora (GH-118983) Co-authored-by: Erlend E. Aasland --- Tools/jit/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 0f5aa9ce656bc9..ae126661c6ce25 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -21,6 +21,12 @@ chmod +x llvm.sh sudo ./llvm.sh 18 ``` +Install LLVM 18 on Fedora Linux 40 or newer: + +```sh +sudo dnf install 'clang(major) = 18' 'llvm(major) = 18' +``` + ### macOS Install LLVM 18 with [Homebrew](https://brew.sh): From 4702b7b5bdc07d046576b4126cf4e4f5f7145abb Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 16 May 2024 12:11:42 -0400 Subject: [PATCH 087/903] GH-118943: Fix a race condition when generating jit_stencils.h (GH-118957) --- ...-05-11-15-11-30.gh-issue-118943.VI_MnY.rst | 3 +++ Tools/jit/_targets.py | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst diff --git a/Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst b/Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst new file mode 100644 index 00000000000000..4e886be034fb82 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst @@ -0,0 +1,3 @@ +Fix a possible race condition affecting parallel builds configured with +``--enable-experimental-jit``, in which compilation errors could be caused +by an incompletely-generated header file. diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index b020f49cf4a2c1..5604c429bcf8ad 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -212,13 +212,18 @@ def build( ): return stencil_groups = asyncio.run(self._build_stencils()) - with jit_stencils.open("w") as file: - file.write(digest) - if comment: - file.write(f"// {comment}\n\n") - file.write("") - for line in _writer.dump(stencil_groups): - file.write(f"{line}\n") + jit_stencils_new = out / "jit_stencils.h.new" + try: + with jit_stencils_new.open("w") as file: + file.write(digest) + if comment: + file.write(f"// {comment}\n") + file.write("\n") + for line in _writer.dump(stencil_groups): + file.write(f"{line}\n") + jit_stencils_new.replace(jit_stencils) + finally: + jit_stencils_new.unlink(missing_ok=True) class _COFF( From 100c7ab00ab66a8c0d54582f35e38d8eb691743c Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 16 May 2024 23:27:59 +0300 Subject: [PATCH 088/903] gh-119049: Fix incorrect display of warning which is constructed by C API (GH-119063) The source line was not displayed if the warnings module had not yet been imported. --- Lib/test/test_capi/test_exceptions.py | 44 ++++++++++++++++++- ...-05-16-23-02-03.gh-issue-119049.qpd_S-.rst | 2 + Modules/_testcapimodule.c | 10 +++++ Python/_warnings.c | 5 +-- 4 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-16-23-02-03.gh-issue-119049.qpd_S-.rst diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index 1d158e3586e98d..c475b6d78d0c56 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -3,11 +3,12 @@ import re import sys import unittest +import textwrap from test import support from test.support import import_helper from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE -from test.support.script_helper import assert_python_failure +from test.support.script_helper import assert_python_failure, assert_python_ok from test.support.testcase import ExceptionIsLikeMixin from .test_misc import decode_stderr @@ -68,6 +69,47 @@ def test_exc_info(self): else: self.assertTrue(False) + def test_warn_with_stacklevel(self): + code = textwrap.dedent('''\ + import _testcapi + + def foo(): + _testcapi.function_set_warning() + + foo() # line 6 + + + foo() # line 9 + ''') + proc = assert_python_ok("-c", code) + warnings = proc.err.splitlines() + self.assertEqual(warnings, [ + b':6: RuntimeWarning: Testing PyErr_WarnEx', + b' foo() # line 6', + b':9: RuntimeWarning: Testing PyErr_WarnEx', + b' foo() # line 9', + ]) + + def test_warn_during_finalization(self): + code = textwrap.dedent('''\ + import _testcapi + + class Foo: + def foo(self): + _testcapi.function_set_warning() + def __del__(self): + self.foo() + + ref = Foo() + ''') + proc = assert_python_ok("-c", code) + warnings = proc.err.splitlines() + # Due to the finalization of the interpreter, the source will be ommited + # because the ``warnings`` module cannot be imported at this time + self.assertEqual(warnings, [ + b':7: RuntimeWarning: Testing PyErr_WarnEx', + ]) + class Test_FatalError(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-16-23-02-03.gh-issue-119049.qpd_S-.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-16-23-02-03.gh-issue-119049.qpd_S-.rst new file mode 100644 index 00000000000000..1d7aad8d1e5be6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-16-23-02-03.gh-issue-119049.qpd_S-.rst @@ -0,0 +1,2 @@ +Fix displaying the source line for warnings created by the C API if the +:mod:`warnings` module had not yet been imported. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ff31724c0e9ff9..f99ebf0dde4f9e 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3303,6 +3303,15 @@ test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) return NULL; } +static PyObject * +function_set_warning(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + if (PyErr_WarnEx(PyExc_RuntimeWarning, "Testing PyErr_WarnEx", 2)) { + return NULL; + } + Py_RETURN_NONE; +} + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -3444,6 +3453,7 @@ static PyMethodDef TestMethods[] = { {"function_set_closure", function_set_closure, METH_VARARGS, NULL}, {"check_pyimport_addmodule", check_pyimport_addmodule, METH_VARARGS}, {"test_weakref_capi", test_weakref_capi, METH_NOARGS}, + {"function_set_warning", function_set_warning, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/_warnings.c b/Python/_warnings.c index 793cbc657f3184..17404d33c1cc9b 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -569,10 +569,9 @@ call_show_warning(PyThreadState *tstate, PyObject *category, PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; PyInterpreterState *interp = tstate->interp; - /* If the source parameter is set, try to get the Python implementation. - The Python implementation is able to log the traceback where the source + /* The Python implementation is able to log the traceback where the source was allocated, whereas the C implementation doesn't. */ - show_fn = GET_WARNINGS_ATTR(interp, _showwarnmsg, source != NULL); + show_fn = GET_WARNINGS_ATTR(interp, _showwarnmsg, 1); if (show_fn == NULL) { if (PyErr_Occurred()) return -1; From 033f5c87f1f876088701d1ae078dc39c41177d4a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 17 May 2024 06:13:24 -0400 Subject: [PATCH 089/903] Improve `pyrepl` type-annotation coverage (#119081) --- Lib/_pyrepl/_minimal_curses.py | 2 +- Lib/_pyrepl/input.py | 4 ++-- Lib/_pyrepl/keymap.py | 2 +- Lib/_pyrepl/pager.py | 8 ++++++-- Lib/_pyrepl/readline.py | 7 +++++-- Lib/_pyrepl/unix_console.py | 22 ++++++++++++++++------ 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py index 0757fb2c664add..849617bf7585e4 100644 --- a/Lib/_pyrepl/_minimal_curses.py +++ b/Lib/_pyrepl/_minimal_curses.py @@ -17,7 +17,7 @@ class error(Exception): pass -def _find_clib(): +def _find_clib() -> str: trylibs = ["ncursesw", "ncurses", "curses"] for lib in trylibs: diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py index 300e16d1d25441..21c24eb5cde3e3 100644 --- a/Lib/_pyrepl/input.py +++ b/Lib/_pyrepl/input.py @@ -60,7 +60,7 @@ def empty(self) -> bool: class KeymapTranslator(InputTranslator): - def __init__(self, keymap, verbose=0, invalid_cls=None, character_cls=None): + def __init__(self, keymap, verbose=False, invalid_cls=None, character_cls=None): self.verbose = verbose from .keymap import compile_keymap, parse_keys @@ -110,5 +110,5 @@ def get(self): else: return None - def empty(self): + def empty(self) -> bool: return not self.results diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py index 31a02642ce8ceb..e1421730e75717 100644 --- a/Lib/_pyrepl/keymap.py +++ b/Lib/_pyrepl/keymap.py @@ -187,7 +187,7 @@ def _parse_key1(key, s): return ret, s -def parse_keys(key): +def parse_keys(key: str) -> list[str]: s = 0 r = [] while s < len(key): diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index af0409c4523bc2..6a076b5181d872 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -76,10 +76,14 @@ def tty_pager(text: str, title: str = '') -> None: fd = sys.stdin.fileno() old = termios.tcgetattr(fd) tty.setcbreak(fd) - getchar = lambda: sys.stdin.read(1) has_tty = True + + def getchar() -> str: + return sys.stdin.read(1) + except (ImportError, AttributeError, io.UnsupportedOperation): - getchar = lambda: sys.stdin.readline()[:-1][:1] + def getchar() -> str: + return sys.stdin.readline()[:-1][:1] try: try: diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index d28a7f3779f302..0adecf235a4eb4 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -48,6 +48,9 @@ from .types import Callback, Completer, KeySpec, CommandName +MoreLinesCallable = Callable[[str], bool] + + __all__ = [ "add_history", "clear_history", @@ -94,7 +97,7 @@ class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): # Instance fields config: ReadlineConfig - more_lines: Callable[[str], bool] | None = None + more_lines: MoreLinesCallable | None = None def __post_init__(self) -> None: super().__post_init__() @@ -287,7 +290,7 @@ def input(self, prompt: object = "") -> str: reader.ps1 = str(prompt) return reader.readline(startup_hook=self.startup_hook) - def multiline_input(self, more_lines, ps1, ps2): + def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> tuple[str, bool]: """Read an input on possibly multiple lines, asking for more lines as long as 'more_lines(unicodetext)' returns an object whose boolean value is true. diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 605318c82ae2ea..7c59f48df406e6 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -40,9 +40,13 @@ from .utils import wlen +TYPE_CHECKING = False + # types -if False: - from typing import IO +if TYPE_CHECKING: + from typing import IO, Literal, overload +else: + overload = lambda func: None class InvalidTerminal(RuntimeError): @@ -157,7 +161,13 @@ def __init__( curses.setupterm(term or None, self.output_fd) self.term = term - def _my_getstr(cap, optional=0): + @overload + def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + + @overload + def _my_getstr(cap: str, optional: bool) -> bytes | None: ... + + def _my_getstr(cap: str, optional: bool = False) -> bytes | None: r = curses.tigetstr(cap) if not optional and r is None: raise InvalidTerminal( @@ -672,18 +682,18 @@ def __move_y_cuu_cud(self, y): elif dy < 0: self.__write_code(self._cuu, -dy) - def __move_x_hpa(self, x): + def __move_x_hpa(self, x: int) -> None: if x != self.__posxy[0]: self.__write_code(self._hpa, x) - def __move_x_cub1_cuf1(self, x): + def __move_x_cub1_cuf1(self, x: int) -> None: dx = x - self.__posxy[0] if dx > 0: self.__write_code(self._cuf1 * dx) elif dx < 0: self.__write_code(self._cub1 * (-dx)) - def __move_x_cub_cuf(self, x): + def __move_x_cub_cuf(self, x: int) -> None: dx = x - self.__posxy[0] if dx > 0: self.__write_code(self._cuf, dx) From 65de194dd80bbc8cb7098d21cfd6aefd11d0d0ce Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Fri, 17 May 2024 18:37:35 +0800 Subject: [PATCH 090/903] Fix typos in documentation (#119092) Co-authored-by: Alex Waygood --- Misc/NEWS.d/3.10.0a4.rst | 2 +- Misc/NEWS.d/3.11.0a1.rst | 2 +- Misc/NEWS.d/3.11.0a2.rst | 2 +- Misc/NEWS.d/3.11.0a5.rst | 2 +- Misc/NEWS.d/3.11.0a6.rst | 8 ++++---- Misc/NEWS.d/3.11.0a7.rst | 4 ++-- Misc/NEWS.d/3.8.0a1.rst | 2 +- Misc/NEWS.d/3.8.0a4.rst | 4 ++-- Misc/NEWS.d/3.9.0a1.rst | 2 +- Misc/NEWS.d/3.9.0a6.rst | 2 +- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Misc/NEWS.d/3.10.0a4.rst b/Misc/NEWS.d/3.10.0a4.rst index 398f7e5d3422cb..ae667f2bffe192 100644 --- a/Misc/NEWS.d/3.10.0a4.rst +++ b/Misc/NEWS.d/3.10.0a4.rst @@ -412,7 +412,7 @@ be created automatically. ``logging.disable`` will now validate the types and value of its parameter. It also now accepts strings representing the levels (as does -``loging.setLevel``) instead of only the numerical values. +``logging.setLevel``) instead of only the numerical values. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 96a7cf6984dd9e..40fbb9d42b7944 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -972,7 +972,7 @@ manager` protocols correspondingly. .. section: Core and Builtins Make sure that the line number is set when entering a comprehension scope. -Ensures that backtraces inclusing generator expressions show the correct +This ensures that backtraces including generator expressions show the correct line number. .. diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index f3fc62e9097162..05644d0a4639b1 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -34,7 +34,7 @@ module but frozen modules are disabled. .. nonce: veL4lJ .. section: Core and Builtins -Specialize simple calls to Python functions (no starargs, keyowrd dict, or +Specialize simple calls to Python functions (no starargs, keyword dict, or closure) .. diff --git a/Misc/NEWS.d/3.11.0a5.rst b/Misc/NEWS.d/3.11.0a5.rst index 30a462e9bfdcbf..954f5c18b48000 100644 --- a/Misc/NEWS.d/3.11.0a5.rst +++ b/Misc/NEWS.d/3.11.0a5.rst @@ -748,7 +748,7 @@ tests to use ``support.infinite_recursion()``. Patch by Victor Stinner. Skip test_builtin PTY tests on non-ASCII characters if the readline module is loaded. The readline module changes input() behavior, but test_builtin is -not intented to test the readline module. Patch by Victor Stinner. +not intended to test the readline module. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index 2fdceef7746d4e..66ffa4ffba52e5 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -1088,7 +1088,7 @@ Patch by Kumar Aditya. Fix wasm32-emscripten test failures and platform issues. - Disable syscalls that are not supported or don't work, e.g. wait, getrusage, prlimit, -mkfifo, mknod, setres[gu]id, setgroups. - Use fd_count to cound open fds. - +mkfifo, mknod, setres[gu]id, setgroups. - Use fd_count to count open fds. - Add more checks for subprocess and fork. - Add workarounds for missing _multiprocessing and failing socket.accept(). - Enable bzip2. - Disable large file support. - Disable signal.alarm. @@ -1162,7 +1162,7 @@ Terry Jan Reedy. .. section: C API Python's public headers no longer import ````, leaving code that -embedd/extends Python free to define ``bool``, ``true`` and ``false``. +embeds/extends Python free to define ``bool``, ``true`` and ``false``. .. @@ -1182,7 +1182,7 @@ internal C API ``pycore_frame.h`` header file. Patch by Victor Stinner. .. section: C API Rename ``Include/buffer.h`` header file to ``Include/pybuffer.h`` to avoid -conflits with projects having an existing ``buffer.h`` header file. Patch by +conflicts with projects having an existing ``buffer.h`` header file. Patch by Victor Stinner. .. @@ -1202,5 +1202,5 @@ API). Patch by Victor Stinner. .. nonce: __ZdpH .. section: C API -Added function :c:func:`PyType_GetModuleByDef`, which allows accesss to +Added function :c:func:`PyType_GetModuleByDef`, which allows access to module state when a method's defining class is not available. diff --git a/Misc/NEWS.d/3.11.0a7.rst b/Misc/NEWS.d/3.11.0a7.rst index f4e2ad8db678f5..a376c8becea9f4 100644 --- a/Misc/NEWS.d/3.11.0a7.rst +++ b/Misc/NEWS.d/3.11.0a7.rst @@ -1173,7 +1173,7 @@ implemented. .. section: Library Add an Barrier object in synchronization primitives of *asyncio* Lib in -order to be consistant with Barrier from *threading* and *multiprocessing* +order to be consistent with Barrier from *threading* and *multiprocessing* libs* .. @@ -1211,7 +1211,7 @@ Update PEP URLs to :pep:`676`'s new canonical form. .. nonce: 4Dn48U .. section: Documentation -Clarified the old Python versions compatiblity note of +Clarified the old Python versions compatibility note of :func:`binascii.crc32` / :func:`zlib.adler32` / :func:`zlib.crc32` functions. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 258e7d82b3f303..9decc4034d6b87 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -6282,7 +6282,7 @@ Add documentation about the new command line interface of the gzip module. .. nonce: YO9CYm .. section: Documentation -chm document displays non-ASCII charaters properly on some MBCS Windows +chm document displays non-ASCII characters properly on some MBCS Windows systems. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 38fa1324dceb40..7bf0de1210935b 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -600,7 +600,7 @@ exceptions. .. nonce: 9sjd38 .. section: Library -Add time module support and fix test_time faiures for VxWorks. +Add time module support and fix test_time failures for VxWorks. .. @@ -843,7 +843,7 @@ Using the code of the ``Tools/scripts/serve.py`` script as an example in the .. nonce: nF1pP1 .. section: Documentation -Added Documention for PyInterpreterState_Main(). +Added documentation for PyInterpreterState_Main(). .. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 39d760cdd4fddf..a38b93e4b76d17 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -1396,7 +1396,7 @@ way to :func:`email.message.get`. .. section: Library Deprecated the ``split()`` method in :class:`_tkinter.TkappType` in favour -of the ``splitlist()`` method which has more consistent and predicable +of the ``splitlist()`` method which has more consistent and predictable behavior. .. diff --git a/Misc/NEWS.d/3.9.0a6.rst b/Misc/NEWS.d/3.9.0a6.rst index 466ff624fcbf81..b7ea1051c314f2 100644 --- a/Misc/NEWS.d/3.9.0a6.rst +++ b/Misc/NEWS.d/3.9.0a6.rst @@ -635,7 +635,7 @@ script is killed by signal 11, it now logs: "CGI script exit code -11." .. section: Library Improve the error message when triying to import a module using :mod:`runpy` -and incorrently use the ".py" extension at the end of the module name. Patch +and incorrectly using the ".py" extension at the end of the module name. Patch by Pablo Galindo. .. From 447edb6e987d22c91f6dfad043f3472ce07bdfc0 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Fri, 17 May 2024 12:10:21 -0400 Subject: [PATCH 091/903] gh-112066: Fix versionadded in PyDict_SetDefaultRef docs (#118696) --- Doc/c-api/dict.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 03f3d28187bfe9..49a78583a6fe26 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -191,6 +191,7 @@ Dictionary Objects to both *default_value* and *\*result* (if it's not ``NULL``). These may refer to the same object: in that case you hold two separate references to it. + .. versionadded:: 3.13 From 31a28cbae0989f57ad01b428c007dade24d9593a Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Fri, 17 May 2024 19:12:02 +0300 Subject: [PATCH 092/903] gh-119049: Defer `import warnings` in `pathlib._local` (#119111) --- Lib/pathlib/_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 011144a565540f..f2776b1d20a2ea 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -4,7 +4,6 @@ import os import posixpath import sys -import warnings from glob import _StringGlobber from itertools import chain from _collections_abc import Sequence @@ -405,6 +404,7 @@ def is_absolute(self): def is_reserved(self): """Return True if the path contains one of the special names reserved by the system, if any.""" + import warnings msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " "for removal in Python 3.15. Use os.path.isreserved() to " "detect reserved paths on Windows.") From 81c3130c51a2b1504842cb1a93732cc03ddbbd79 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 18 May 2024 01:32:34 -0500 Subject: [PATCH 093/903] Minor improvements to the docs for itertools.tee() (gh-119135) --- Doc/library/itertools.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index a19baa3f0e439f..6d33748898361d 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -698,18 +698,19 @@ loops that truncate the stream. def tee(iterable, n=2): iterator = iter(iterable) - empty_link = [None, None] # Singly linked list: [value, link] - return tuple(_tee(iterator, empty_link) for _ in range(n)) + shared_link = [None, None] + return tuple(_tee(iterator, shared_link) for _ in range(n)) def _tee(iterator, link): - while True: - if link[1] is None: - try: - link[:] = [next(iterator), [None, None]] - except StopIteration: - return - value, link = link - yield value + try: + while True: + if link[1] is None: + link[0] = next(iterator) + link[1] = [None, None] + value, link = link + yield value + except StopIteration: + return Once a :func:`tee` has been created, the original *iterable* should not be used anywhere else; otherwise, the *iterable* could get advanced without From 0f5e8bed636c2f29701e5a1965d1b088d33abbf0 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Sat, 18 May 2024 13:44:02 +0200 Subject: [PATCH 094/903] gh-119078: Clarify venv tutorial (GH-119129) --- Doc/tutorial/venv.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index a6dead2eac11f6..6cca3f1b25aadc 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -36,10 +36,10 @@ Creating Virtual Environments ============================= The module used to create and manage virtual environments is called -:mod:`venv`. :mod:`venv` will usually install the most recent version of -Python that you have available. If you have multiple versions of Python on your -system, you can select a specific Python version by running ``python3`` or -whichever version you want. +:mod:`venv`. :mod:`venv` will install the Python version from which +the command was run (as reported by the :option:`--version` option). +For instance, excuting the command with ``python3.12`` will install +version 3.12. To create a virtual environment, decide upon a directory where you want to place it, and run the :mod:`venv` module as a script with the directory path:: From 74072a3ffc733e32159e694bcf7a2198f2db0d43 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 18 May 2024 09:24:22 -0400 Subject: [PATCH 095/903] gh-119132: Log sys._is_gil_enabled() in test.pythoninfo (#119140) --- Lib/test/pythoninfo.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 1db9fb9537f888..d928e002ebda10 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -105,9 +105,13 @@ def collect_sys(info_add): ) copy_attributes(info_add, sys, 'sys.%s', attributes) - call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel') - call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion') - call_func(info_add, 'sys.getrecursionlimit', sys, 'getrecursionlimit') + for func in ( + '_is_gil_enabled', + 'getandroidapilevel', + 'getrecursionlimit', + 'getwindowsversion', + ): + call_func(info_add, f'sys.{func}', sys, func) encoding = sys.getfilesystemencoding() if hasattr(sys, 'getfilesystemencodeerrors'): From 691429702f1cb657e65f4e5275bb5ed16121d2b7 Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Sat, 18 May 2024 13:22:54 -0300 Subject: [PATCH 096/903] docs: make mimalloc license text literal (#119046) --- Doc/license.rst | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Doc/license.rst b/Doc/license.rst index 814b6829f6b2bd..674ac5f56e6f97 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -1047,27 +1047,27 @@ https://www.w3.org/TR/xml-c14n2-testcases/ and is distributed under the mimalloc -------- -MIT License - -Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +MIT License:: + + Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. asyncio From c141d4393750c827cbcb3867f0f42997a3bb3528 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sat, 18 May 2024 15:44:40 -0400 Subject: [PATCH 097/903] gh-119132: Update sys.version to identify free-threaded or not. (gh-119134) --- Lib/platform.py | 24 ++++++++++++------- ...-05-17-19-53-27.gh-issue-119132.wepPgM.rst | 2 ++ Python/getversion.c | 9 +++++-- 3 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-05-17-19-53-27.gh-issue-119132.wepPgM.rst diff --git a/Lib/platform.py b/Lib/platform.py index ebaba37563120e..5958382276e79c 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1153,17 +1153,16 @@ def _sys_version(sys_version=None): if result is not None: return result - sys_version_parser = re.compile( - r'([\w.+]+)\s*' # "version" - r'\(#?([^,]+)' # "(#buildno" - r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" - r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" - if sys.platform.startswith('java'): # Jython + jython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" name = 'Jython' - match = sys_version_parser.match(sys_version) + match = jython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % @@ -1190,7 +1189,14 @@ def _sys_version(sys_version=None): else: # CPython - match = sys_version_parser.match(sys_version) + cpython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'(?:experimental free-threading build\s+)?' # "free-threading-build" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" + match = cpython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % diff --git a/Misc/NEWS.d/next/Build/2024-05-17-19-53-27.gh-issue-119132.wepPgM.rst b/Misc/NEWS.d/next/Build/2024-05-17-19-53-27.gh-issue-119132.wepPgM.rst new file mode 100644 index 00000000000000..44fe2a1a1f6725 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-17-19-53-27.gh-issue-119132.wepPgM.rst @@ -0,0 +1,2 @@ +Update :data:`sys.version` to identify whether the build is default build or +free-threading build. Patch By Donghee Na. diff --git a/Python/getversion.c b/Python/getversion.c index 5db836ab4bfd6d..226b2f999a6bfd 100644 --- a/Python/getversion.c +++ b/Python/getversion.c @@ -6,7 +6,7 @@ #include "patchlevel.h" static int initialized = 0; -static char version[250]; +static char version[300]; void _Py_InitVersion(void) { @@ -14,7 +14,12 @@ void _Py_InitVersion(void) return; } initialized = 1; - PyOS_snprintf(version, sizeof(version), "%.80s (%.80s) %.80s", +#ifdef Py_GIL_DISABLED + const char *buildinfo_format = "%.80s experimental free-threading build (%.80s) %.80s"; +#else + const char *buildinfo_format = "%.80s (%.80s) %.80s"; +#endif + PyOS_snprintf(version, sizeof(version), buildinfo_format, PY_VERSION, Py_GetBuildInfo(), Py_GetCompiler()); } From 30b4e9f9c42493136c58c56fee5553128bb32428 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 18 May 2024 16:21:05 -0400 Subject: [PATCH 098/903] gh-119050: Add type hints to libregrtest/results.py (#119144) Sort also 'omitted' in TestResults.display_result(). --- Lib/test/libregrtest/results.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 85c82052eae19b..0e28435bc7d629 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -18,7 +18,7 @@ class TestResults: - def __init__(self): + def __init__(self) -> None: self.bad: TestList = [] self.good: TestList = [] self.rerun_bad: TestList = [] @@ -38,22 +38,22 @@ def __init__(self): # used by -T with -j self.covered_lines: set[Location] = set() - def is_all_good(self): + def is_all_good(self) -> bool: return (not self.bad and not self.skipped and not self.interrupted and not self.worker_bug) - def get_executed(self): + def get_executed(self) -> set[TestName]: return (set(self.good) | set(self.bad) | set(self.skipped) | set(self.resource_denied) | set(self.env_changed) | set(self.run_no_tests)) - def no_tests_run(self): + def no_tests_run(self) -> bool: return not any((self.good, self.bad, self.skipped, self.interrupted, self.env_changed)) - def get_state(self, fail_env_changed): + def get_state(self, fail_env_changed: bool) -> str: state = [] if self.bad: state.append("FAILURE") @@ -204,7 +204,7 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): omitted = set(tests) - self.get_executed() # less important - all_tests.append((omitted, "test", "{} omitted:")) + all_tests.append((sorted(omitted), "test", "{} omitted:")) if not quiet: all_tests.append((self.skipped, "test", "{} skipped:")) all_tests.append((self.resource_denied, "test", "{} skipped (resource denied):")) From caf6064a1bc15ac344afd78b780188e60b9c628e Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 18 May 2024 23:40:51 +0100 Subject: [PATCH 099/903] GH-118447: Fix handling of unreadable symlinks in `os.path.realpath()` (#118489) Co-authored-by: Nice Zombies --- Lib/posixpath.py | 26 +++++++++---------- Lib/test/test_posixpath.py | 17 ++++++++++++ ...-05-01-22-24-05.gh-issue-110863.GjYBbq.rst | 2 ++ 3 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-01-22-24-05.gh-issue-110863.GjYBbq.rst diff --git a/Lib/posixpath.py b/Lib/posixpath.py index b4547d7893b1cd..c04c628de55ee2 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -471,26 +471,26 @@ def realpath(filename, *, strict=False): if not stat.S_ISLNK(st.st_mode): path = newpath continue + if newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + # Raise OSError(errno.ELOOP) + os.stat(newpath) + path = newpath + continue + target = os.readlink(newpath) except OSError: if strict: raise path = newpath continue # Resolve the symbolic link - if newpath in seen: - # Already seen this path - path = seen[newpath] - if path is not None: - # use cached value - continue - # The symlink is not resolved, so we must have a symlink loop. - if strict: - # Raise OSError(errno.ELOOP) - os.stat(newpath) - path = newpath - continue seen[newpath] = None # not resolved symlink - target = os.readlink(newpath) if target.startswith(sep): # Symlink target is absolute; reset resolved path. path = sep diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 32a20efbb64e1d..5c27b7bee8f60e 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -660,6 +660,23 @@ def test_realpath_resolve_first(self): safe_rmdir(ABSTFN + "/k") safe_rmdir(ABSTFN) + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + def test_realpath_unreadable_symlink(self): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + self.assertEqual(realpath(ABSTFN), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/foo'), ABSTFN + '/foo') + self.assertEqual(realpath(ABSTFN + '/../foo'), dirname(ABSTFN) + '/foo') + self.assertEqual(realpath(ABSTFN + '/foo/..'), ABSTFN) + with self.assertRaises(PermissionError): + realpath(ABSTFN, strict=True) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os.unlink(ABSTFN) + def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") try: diff --git a/Misc/NEWS.d/next/Library/2024-05-01-22-24-05.gh-issue-110863.GjYBbq.rst b/Misc/NEWS.d/next/Library/2024-05-01-22-24-05.gh-issue-110863.GjYBbq.rst new file mode 100644 index 00000000000000..37e27a6e37c7d0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-01-22-24-05.gh-issue-110863.GjYBbq.rst @@ -0,0 +1,2 @@ +:func:`os.path.realpath` now suppresses any :exc:`OSError` from +:func:`os.readlink` when *strict* mode is disabled (the default). From ecd8664f11298a1a4f7428363c55ad2904c9f279 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 18 May 2024 19:19:57 -0500 Subject: [PATCH 100/903] gh-118750: Asymptotically faster `int(string)` (#118751) Asymptotically faster (O(n log n)) str->int for very large strings, leveraging the faster multiplication scheme in the C-coded `_decimal` when available. This is used instead of the current Karatsuba-limited method starting at 2 million digits. Lots of opportunity remains for fine-tuning. Good targets include changing BYTELIM, and possibly changing the internal output base (from 256 to a higher number of bytes). Doing this was substantial work, and many of the new lines are actually comments giving correctness proofs. The obvious approaches sticking to integers were too slow to be useful, so this is doing variable-precision decimal floating-point arithmetic. Much faster, but worst-possible rounding errors have to be wholly accounted for, using as little precision as possible. Special thanks to Serhiy Storchaka for asking many good questions in his code reviews! Co-authored-by: Jelle Zijlstra Co-authored-by: sstandre <43125375+sstandre@users.noreply.github.com> Co-authored-by: Pieter Eendebak Co-authored-by: Nice Zombies --- Lib/_pylong.py | 432 ++++++++++++++++-- Lib/test/test_int.py | 79 ++++ ...-05-09-02-37-25.gh-issue-118750.7aLfT-.rst | 1 + 3 files changed, 479 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-09-02-37-25.gh-issue-118750.7aLfT-.rst diff --git a/Lib/_pylong.py b/Lib/_pylong.py index 4970eb3fa67b2b..f7aabde1434725 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -45,10 +45,16 @@ # # and `mycache[lo]` replaces `base**lo` in the inner function. # -# While this does give minor speedups (a few percent at best), the primary -# intent is to simplify the functions using this, by eliminating the need for -# them to craft their own ad-hoc caching schemes. -def compute_powers(w, base, more_than, show=False): +# If an algorithm wants the powers of ceiling(w/2) instead of the floor, +# pass keyword argument `need_hi=True`. +# +# While this does give minor speedups (a few percent at best), the +# primary intent is to simplify the functions using this, by eliminating +# the need for them to craft their own ad-hoc caching schemes. +# +# See code near end of file for a block of code that can be enabled to +# run millions of tests. +def compute_powers(w, base, more_than, *, need_hi=False, show=False): seen = set() need = set() ws = {w} @@ -58,40 +64,70 @@ def compute_powers(w, base, more_than, show=False): continue seen.add(w) lo = w >> 1 - # only _need_ lo here; some other path may, or may not, need hi - need.add(lo) - ws.add(lo) - if w & 1: - ws.add(lo + 1) + hi = w - lo + # only _need_ one here; the other may, or may not, be needed + which = hi if need_hi else lo + need.add(which) + ws.add(which) + if lo != hi: + ws.add(w - which) + + # `need` is the set of exponents needed. To compute them all + # efficiently, possibly add other exponents to `extra`. The goal is + # to ensure that each exponent can be gotten from a smaller one via + # multiplying by the base, squaring it, or squaring and then + # multiplying by the base. + # + # If need_hi is False, this is already the case (w can always be + # gotten from w >> 1 via one of the squaring strategies). But we do + # the work anyway, just in case ;-) + # + # Note that speed is irrelevant. These loops are working on little + # ints (exponents) and go around O(log w) times. The total cost is + # insignificant compared to just one of the bigint multiplies. + cands = need.copy() + extra = set() + while cands: + w = max(cands) + cands.remove(w) + lo = w >> 1 + if lo > more_than and w-1 not in cands and lo not in cands: + extra.add(lo) + cands.add(lo) + assert need_hi or not extra d = {} - if not need: - return d - it = iter(sorted(need)) - first = next(it) - if show: - print("pow at", first) - d[first] = base ** first - for this in it: - if this - 1 in d: + for n in sorted(need | extra): + lo = n >> 1 + hi = n - lo + if n-1 in d: if show: - print("* base at", this) - d[this] = d[this - 1] * base # cheap - else: - lo = this >> 1 - hi = this - lo - assert lo in d + print("* base", end="") + result = d[n-1] * base # cheap! + elif lo in d: + # Multiplying a bigint by itself is about twice as fast + # in CPython provided it's the same object. if show: - print("square at", this) - # Multiplying a bigint by itself (same object!) is about twice - # as fast in CPython. - sq = d[lo] * d[lo] + print("square", end="") + result = d[lo] * d[lo] # same object if hi != lo: - assert hi == lo + 1 if show: - print(" and * base") - sq *= base - d[this] = sq + print(" * base", end="") + assert 2 * lo + 1 == n + result *= base + else: # rare + if show: + print("pow", end='') + result = base ** n + if show: + print(" at", n, "needed" if n in need else "extra") + d[n] = result + + assert need <= d.keys() + if excess := d.keys() - need: + assert need_hi + for n in excess: + del d[n] return d _unbounded_dec_context = decimal.getcontext().copy() @@ -211,6 +247,145 @@ def inner(a, b): return inner(0, len(s)) +# Asymptotically faster version, using the C decimal module. See +# comments at the end of the file. This uses decimal arithmetic to +# convert from base 10 to base 256. The latter is just a string of +# bytes, which CPython can convert very efficiently to a Python int. + +# log of 10 to base 256 with best-possible 53-bit precision. Obtained +# via: +# from mpmath import mp +# mp.prec = 1000 +# print(float(mp.log(10, 256)).hex()) +_LOG_10_BASE_256 = float.fromhex('0x1.a934f0979a371p-2') # about 0.415 + +# _spread is for internal testing. It maps a key to the number of times +# that condition obtained in _dec_str_to_int_inner: +# key 0 - quotient guess was right +# key 1 - quotient had to be boosted by 1, one time +# key 999 - one adjustment wasn't enough, so fell back to divmod +from collections import defaultdict +_spread = defaultdict(int) +del defaultdict + +def _dec_str_to_int_inner(s, *, GUARD=8): + # Yes, BYTELIM is "large". Large enough that CPython will usually + # use the Karatsuba _str_to_int_inner to convert the string. This + # allowed reducing the cutoff for calling _this_ function from 3.5M + # to 2M digits. We could almost certainly do even better by + # fine-tuning this and/or using a larger output base than 256. + BYTELIM = 100_000 + D = decimal.Decimal + result = bytearray() + # See notes at end of file for discussion of GUARD. + assert GUARD > 0 # if 0, `decimal` can blow up - .prec 0 not allowed + + def inner(n, w): + #assert n < D256 ** w # required, but too expensive to check + if w <= BYTELIM: + # XXX Stefan Pochmann discovered that, for 1024-bit ints, + # `int(Decimal)` took 2.5x longer than `int(str(Decimal))`. + # Worse, `int(Decimal) is still quadratic-time for much + # larger ints. So unless/until all that is repaired, the + # seemingly redundant `str(Decimal)` is crucial to speed. + result.extend(int(str(n)).to_bytes(w)) # big-endian default + return + w1 = w >> 1 + w2 = w - w1 + if 0: + # This is maximally clear, but "too slow". `decimal` + # division is asymptotically fast, but we have no way to + # tell it to reuse the high-precision reciprocal it computes + # for pow256[w2], so it has to recompute it over & over & + # over again :-( + hi, lo = divmod(n, pow256[w2][0]) + else: + p256, recip = pow256[w2] + # The integer part will have a number of digits about equal + # to the difference between the log10s of `n` and `pow256` + # (which, since these are integers, is roughly approximated + # by `.adjusted()`). That's the working precision we need, + ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD + hi = +n * +recip # unary `+` chops back to ctx.prec digits + ctx.prec = decimal.MAX_PREC + hi = hi.to_integral_value() # lose the fractional digits + lo = n - hi * p256 + # Because we've been uniformly rounding down, `hi` is a + # lower bound on the correct quotient. + assert lo >= 0 + # Adjust quotient up if needed. It usually isn't. In random + # testing on inputs through 5 billion digit strings, the + # test triggered once in about 200 thousand tries. + count = 0 + if lo >= p256: + count = 1 + lo -= p256 + hi += 1 + if lo >= p256: + # Complete correction via an exact computation. I + # believe it's not possible to get here provided + # GUARD >= 3. It's tested by reducing GUARD below + # that. + count = 999 + hi2, lo = divmod(lo, p256) + hi += hi2 + _spread[count] += 1 + # The assert should always succeed, but way too slow to keep + # enabled. + #assert hi, lo == divmod(n, pow256[w2][0]) + inner(hi, w1) + del hi # at top levels, can free a lot of RAM "early" + inner(lo, w2) + + # How many base 256 digits are needed?. Mathematically, exactly + # floor(log256(int(s))) + 1. There is no cheap way to compute this. + # But we can get an upper bound, and that's necessary for our error + # analysis to make sense. int(s) < 10**len(s), so the log needed is + # < log256(10**len(s)) = len(s) * log256(10). However, using + # finite-precision floating point for this, it's possible that the + # computed value is a little less than the true value. If the true + # value is at - or a little higher than - an integer, we can get an + # off-by-1 error too low. So we add 2 instead of 1 if chopping lost + # a fraction > 0.9. + + # The "WASI" test platfrom can complain about `len(s)` if it's too + # large to fit in its idea of "an index-sized integer". + lenS = s.__len__() + log_ub = lenS * _LOG_10_BASE_256 + log_ub_as_int = int(log_ub) + w = log_ub_as_int + 1 + (log_ub - log_ub_as_int > 0.9) + # And what if we've plain exhausted the limits of HW floats? We + # could compute the log to any desired precision using `decimal`, + # but it's not plausible that anyone will pass a string requiring + # trillions of bytes (unless they're just trying to "break things"). + if w.bit_length() >= 46: + # "Only" had < 53 - 46 = 7 bits to spare in IEEE-754 double. + raise ValueError(f"cannot convert string of len {lenS} to int") + with decimal.localcontext(_unbounded_dec_context) as ctx: + D256 = D(256) + pow256 = compute_powers(w, D256, BYTELIM, need_hi=True) + rpow256 = compute_powers(w, 1 / D256, BYTELIM, need_hi=True) + # We're going to do inexact, chopped arithmetic, multiplying by + # an approximation to the reciprocal of 256**i. We chop to get a + # lower bound on the true integer quotient. Our approximation is + # a lower bound, the multiplication is chopped too, and + # to_integral_value() is also chopped. + ctx.traps[decimal.Inexact] = 0 + ctx.rounding = decimal.ROUND_DOWN + for k, v in pow256.items(): + # No need to save much more precision in the reciprocal than + # the power of 256 has, plus some guard digits to absorb + # most relevant rounding errors. This is highly significant: + # 1/2**i has the same number of significant decimal digits + # as 5**i, generally over twice the number in 2**i, + ctx.prec = v.adjusted() + GUARD + 1 + # The unary "+" chops the reciprocal back to that precision. + pow256[k] = v, +rpow256[k] + del rpow256 # exact reciprocals no longer needed + ctx.prec = decimal.MAX_PREC + inner(D(s), w) + return int.from_bytes(result) + def int_from_string(s): """Asymptotically fast version of PyLong_FromString(), conversion of a string of decimal digits into an 'int'.""" @@ -219,7 +394,10 @@ def int_from_string(s): # and underscores, and stripped leading whitespace. The input can still # contain underscores and have trailing whitespace. s = s.rstrip().replace('_', '') - return _str_to_int_inner(s) + func = _str_to_int_inner + if len(s) >= 2_000_000 and _decimal is not None: + func = _dec_str_to_int_inner + return func(s) def str_to_int(s): """Asymptotically fast version of decimal string to 'int' conversion.""" @@ -361,3 +539,191 @@ def int_divmod(a, b): return ~q, b + ~r else: return _divmod_pos(a, b) + + +# Notes on _dec_str_to_int_inner: +# +# Stefan Pochmann worked up a str->int function that used the decimal +# module to, in effect, convert from base 10 to base 256. This is +# "unnatural", in that it requires multiplying and dividing by large +# powers of 2, which `decimal` isn't naturally suited to. But +# `decimal`'s `*` and `/` are asymptotically superior to CPython's, so +# at _some_ point it could be expected to win. +# +# Alas, the crossover point was too high to be of much real interest. I +# (Tim) then worked on ways to replace its division with multiplication +# by a cached reciprocal approximation instead, fixing up errors +# afterwards. This reduced the crossover point significantly, +# +# I revisited the code, and found ways to improve and simplify it. The +# crossover point is at about 3.4 million digits now. +# +# About .adjusted() +# ----------------- +# Restrict to Decimal values x > 0. We don't use negative numbers in the +# code, and I don't want to have to keep typing, e.g., "absolute value". +# +# For convenience, I'll use `x.a` to mean `x.adjusted()`. x.a doesn't +# look at the digits of x, but instead returns an integer giving x's +# order of magnitude. These are equivalent: +# +# - x.a is the power-of-10 exponent of x's most significant digit. +# - x.a = the infinitely precise floor(log10(x)) +# - x can be written in this form, where f is a real with 1 <= f < 10: +# x = f * 10**x.a +# +# Observation; if x is an integer, len(str(x)) = x.a + 1. +# +# Lemma 1: (x * y).a = x.a + y.a, or one larger +# +# Proof: Write x = f * 10**x.a and y = g * 10**y.a, where f and g are in +# [1, 10). Then x*y = f*g * 10**(x.a + y.a), where 1 <= f*g < 100. If +# f*g < 10, (x*y).a is x.a+y.a. Else divide f*g by 10 to bring it back +# into [1, 10], and add 1 to the exponent to compensate. Then (x*y).a is +# x.a+y.a+1. +# +# Lemma 2: ceiling(log10(x/y)) <= x.a - y.a + 1 +# +# Proof: Express x and y as in Lemma 1. Then x/y = f/g * 10**(x.a - +# y.a), where 1/10 < f/g < 10. If 1 <= f/g, (x/y).a is x.a-y.a. Else +# multiply f/g by 10 to bring it back into [1, 10], and subtract 1 from +# the exponent to compensate. Then (x/y).a is x.a-y.a-1. So the largest +# (x/y).a can be is x.a-y.a. Since that's the floor of log10(x/y). the +# ceiling is at most 1 larger (with equality iff f/g = 1 exactly). +# +# GUARD digits +# ------------ +# We only want the integer part of divisions, so don't need to build +# the full multiplication tree. But using _just_ the number of +# digits expected in the integer part ignores too much. What's left +# out can have a very significant effect on the quotient. So we use +# GUARD additional digits. +# +# The default 8 is more than enough so no more than 1 correction step +# was ever needed for all inputs tried through 2.5 billion digits. In +# fact, I believe 3 guard digits are always enough - but the proof is +# very involved, so better safe than sorry. +# +# Short course: +# +# If prec is the decimal precision in effect, and we're rounding down, +# the result of an operation is exactly equal to the infinitely precise +# result times 1-e for some real e with 0 <= e < 10**(1-prec). In +# +# ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD +# hi = +n * +recip # unary `+` chops to ctx.prec digits +# +# we have 3 visible chopped operationa, but there's also a 4th: +# precomputing a truncated `recip` as part of setup. +# +# So the computed product is exactly equal to the true product times +# (1-e1)*(1-e2)*(1-e3)*(1-e4); since the e's are all very small, an +# excellent approximation to the second factor is 1-(e1+e2+e3+e4) (the +# 2nd and higher order terms in the expanded product are too tiny to +# matter). If they're all as large as possible, that's +# +# 1 - 4*10**(1-prec). This, BTW, is all bog-standard FP error analysis. +# +# That implies the computed product is within 1 of the true product +# provided prec >= log10(true_product) + 1.602. +# +# Here are telegraphic details, rephrasing the initial condition in +# equivalent ways, step by step: +# +# prod - prod * (1 - 4*10**(1-prec)) <= 1 +# prod - prod + prod * 4*10**(1-prec)) <= 1 +# prod * 4*10**(1-prec)) <= 1 +# 10**(log10(prod)) * 4*10**(1-prec)) <= 1 +# 4*10**(1-prec+log10(prod))) <= 1 +# 10**(1-prec+log10(prod))) <= 1/4 +# 1-prec+log10(prod) <= log10(1/4) = -0.602 +# -prec <= -1.602 - log10(prod) +# prec >= log10(prod) + 1.602 +# +# The true product is the same as the true ratio n/p256. By Lemma 2 +# above, n.a - p256.a + 1 is an upper bound on the ceiling of +# log10(prod). Then 2 is the ceiling of 1.602. so n.a - p256.a + 3 is an +# upper bound on the right hand side of the inequality. Any prec >= that +# will work. +# +# But since this is just a sketch of a proof ;-), the code uses the +# empirically tested 8 instead of 3. 5 digits more or less makes no +# practical difference to speed - these ints are huge. And while +# increasing GUARD above 3 may not be necessary, every increase cuts the +# percentage of cases that need a correction at all. +# +# On Computing Reciprocals +# ------------------------ +# In general, the exact reciprocals we compute have over twice as many +# significant digits as needed. 1/256**i has the same number of +# significant decimal digits as 5**i. It's a significant waste of RAM +# to store all those unneeded digits. +# +# So we cut exact reciprocals back to the least precision that can +# be needed so that the error analysis above is valid, +# +# [Note: turns out it's very significantly faster to do it this way than +# to compute 1 / 256**i directly to the desired precision, because the +# power method doesn't require division. It's also faster than computing +# (1/256)**i directly to the desired precision - no material division +# there, but `compute_powers()` is much smarter about _how_ to compute +# all the powers needed than repeated applications of `**` - that +# function invokes `**` for at most the few smallest powers needed.] +# +# The hard part is that chopping back to a shorter width occurs +# _outside_ of `inner`. We can't know then what `prec` `inner()` will +# need. We have to pick, for each value of `w2`, the largest possible +# value `prec` can become when `inner()` is working on `w2`. +# +# This is the `prec` inner() uses: +# max(n.a - p256.a, 0) + GUARD +# and what setup uses (renaming its `v` to `p256` - same thing): +# p256.a + GUARD + 1 +# +# We need that the second is always at least as large as the first, +# which is the same as requiring +# +# n.a - 2 * p256.a <= 1 +# +# What's the largest n can be? n < 255**w = 256**(w2 + (w - w2)). The +# worst case in this context is when w ix even. and then w = 2*w2, so +# n < 256**(2*w2) = (256**w2)**2 = p256**2. By Lemma 1, then, n.a +# is at most p256.a + p256.a + 1. +# +# So the most n.a - 2 * p256.a can be is +# p256.a + p256.a + 1 - 2 * p256.a = 1. QED +# +# Note: an earlier version of the code split on floor(e/2) instead of on +# the ceiling. The worst case then is odd `w`, and a more involved proof +# was needed to show that adding 4 (instead of 1) may be necessary. +# Basically because, in that case, n may be up to 256 times larger than +# p256**2. Curiously enough, by splitting on the ceiling instead, +# nothing in any proof here actually depends on the output base (256). + +# Enable for brute-force testing of compute_powers(). This takes about a +# minute, because it tries millions of cases. +if 0: + def consumer(w, limir, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit: + return + if w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + exp = compute_powers(w, 1, limir, need_hi=need_hi) + assert exp.keys() == need + + from itertools import chain + for need_hi in (False, True): + for limit in (0, 1, 10, 100, 1_000, 10_000, 100_000): + for w in chain(range(1, 100_000), + (10**i for i in range(5, 30))): + consumer(w, limit, need_hi) diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 8959ffb6dcc236..67c080117edcc3 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -919,5 +919,84 @@ def test_pylong_roundtrip(self): self.assertEqual(n, int(sn)) bits <<= 1 + @support.requires_resource('cpu') + def test_pylong_roundtrip_huge(self): + # k blocks of 1234567890 + k = 1_000_000 # so 10 million digits in all + tentoten = 10**10 + n = 1234567890 * ((tentoten**k - 1) // (tentoten - 1)) + sn = "1234567890" * k + self.assertEqual(n, int(sn)) + self.assertEqual(sn, str(n)) + + @support.requires_resource('cpu') + @unittest.skipUnless(_pylong, "_pylong module required") + def test_whitebox_dec_str_to_int_inner_failsafe(self): + # While I believe the number of GUARD digits in this function is + # always enough so that no more than one correction step is ever + # needed, the code has a "failsafe" path that takes over if I'm + # wrong about that. We have no input that reaches that block. + # Here we test a contrived input that _does_ reach that block, + # provided the number of guard digits is reduced to 1. + sn = "9" * 2000156 + n = 10**len(sn) - 1 + orig_spread = _pylong._spread.copy() + _pylong._spread.clear() + try: + self.assertEqual(n, _pylong._dec_str_to_int_inner(sn, GUARD=1)) + self.assertIn(999, _pylong._spread) + finally: + _pylong._spread.clear() + _pylong._spread.update(orig_spread) + + @unittest.skipUnless(_pylong, "pylong module required") + def test_whitebox_dec_str_to_int_inner_monster(self): + # I don't think anyone has enough RAM to build a string long enough + # for this function to complain. So lie about the string length. + + class LyingStr(str): + def __len__(self): + return int((1 << 47) / _pylong._LOG_10_BASE_256) + + liar = LyingStr("42") + # We have to pass the liar directly to the complaining function. If we + # just try `int(liar)`, earlier layers will replace it with plain old + # "43". + # Embedding `len(liar)` into the f-string failed on the WASI testbot + # (don't know what that is): + # OverflowError: cannot fit 'int' into an index-sized integer + # So a random stab at worming around that. + self.assertRaisesRegex(ValueError, + f"^cannot convert string of len {liar.__len__()} to int$", + _pylong._dec_str_to_int_inner, + liar) + + @unittest.skipUnless(_pylong, "_pylong module required") + def test_pylong_compute_powers(self): + # Basic sanity tests. See end of _pylong.py for manual heavy tests. + def consumer(w, base, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit or w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + d = _pylong.compute_powers(w, base, limit, need_hi=need_hi) + self.assertEqual(d.keys(), need) + for k, v in d.items(): + self.assertEqual(v, base ** k) + + for base in 2, 5: + for need_hi in False, True: + for limit in 1, 11: + for w in range(250, 550): + consumer(w, base, limit, need_hi) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-09-02-37-25.gh-issue-118750.7aLfT-.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-09-02-37-25.gh-issue-118750.7aLfT-.rst new file mode 100644 index 00000000000000..727427d451d1e0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-09-02-37-25.gh-issue-118750.7aLfT-.rst @@ -0,0 +1 @@ +If the C version of the ``decimal`` module is available, ``int(str)`` now uses it to supply an asymptotically much faster conversion. However, this only applies if the string contains over about 2 million digits. From ba8af848648d3eb51eb17395e12117007bae8606 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 18 May 2024 20:54:23 -0500 Subject: [PATCH 101/903] Try to repair oddball test bots timing out in test_int (#119166) Various test bots (outside the ones GH normally runs) are timing out during test_int after ecd8664 (asymptotically faster str->int). Best guess is that they don't build the C _decimal module. So require that module in the most likely tests to time out then. Flying mostly blind, though! --- Lib/test/test_int.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 67c080117edcc3..caeccbe1fed026 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -12,6 +12,11 @@ except ImportError: _pylong = None +try: + import _decimal +except ImportError: + _decimal = None + L = [ ('0', 0), ('1', 1), @@ -920,6 +925,7 @@ def test_pylong_roundtrip(self): bits <<= 1 @support.requires_resource('cpu') + @unittest.skipUnless(_decimal, "C _decimal module required") def test_pylong_roundtrip_huge(self): # k blocks of 1234567890 k = 1_000_000 # so 10 million digits in all @@ -931,6 +937,7 @@ def test_pylong_roundtrip_huge(self): @support.requires_resource('cpu') @unittest.skipUnless(_pylong, "_pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") def test_whitebox_dec_str_to_int_inner_failsafe(self): # While I believe the number of GUARD digits in this function is # always enough so that no more than one correction step is ever @@ -950,6 +957,7 @@ def test_whitebox_dec_str_to_int_inner_failsafe(self): _pylong._spread.update(orig_spread) @unittest.skipUnless(_pylong, "pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") def test_whitebox_dec_str_to_int_inner_monster(self): # I don't think anyone has enough RAM to build a string long enough # for this function to complain. So lie about the string length. From 697465ff88e49d98443025474e5b534adfba2cb0 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 18 May 2024 22:15:14 -0400 Subject: [PATCH 102/903] marshal docs: Remove reference to "Sun" (#119161) Nobody has been using a Sun machine for a long time. When I saw this sentence in a lightning talk just now, I thought it was talking about sending Python code on a spacecraft. --- Doc/library/marshal.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index c8f1d57317ea68..f9ba4d554b0c22 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -10,7 +10,7 @@ This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, -transport the file to a Sun, and read it back there). Details of the format are +transport the file to a Mac, and read it back there). Details of the format are undocumented on purpose; it may change between Python versions (although it rarely does). [#]_ From 5307f44fb983f2a17727fb43602f5dfa63e93311 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sun, 19 May 2024 08:59:12 -0400 Subject: [PATCH 103/903] GH-119146: Don't run JIT CI on unrelated changes (GH-119147) Co-authored-by: Alex Waygood --- .github/workflows/jit.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 7152cde8f4607c..2ec04da16946ff 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -5,11 +5,17 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' + - '!Python/perf_jit_trampoline.c' + - '!**/*.md' + - '!**/*.ini' push: paths: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' + - '!Python/perf_jit_trampoline.c' + - '!**/*.md' + - '!**/*.ini' workflow_dispatch: permissions: From 4b7667172898d440c1931ae923446c6a5ef1765e Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 19 May 2024 16:39:00 +0100 Subject: [PATCH 104/903] GH-118447: Fix FreeBSD test failures. (#119170) Apparently only macOS requires read permission to call `readlink()` on a symlink. --- Lib/test/test_posixpath.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 5c27b7bee8f60e..238baed5efa264 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -663,6 +663,7 @@ def test_realpath_resolve_first(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") def test_realpath_unreadable_symlink(self): try: os.symlink(ABSTFN+"1", ABSTFN) From 3c28510b984392b8dac87a17dfc5887366d5c4ab Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 19 May 2024 17:04:56 +0100 Subject: [PATCH 105/903] GH-119113: Raise `TypeError` from `pathlib.PurePath.with_suffix(None)` (#119124) Restore behaviour from 3.12 when `path.with_suffix(None)` is called. --- Lib/pathlib/_abc.py | 10 ++++------ Lib/test/test_pathlib/test_pathlib_abc.py | 4 +++- .../2024-05-17-17-32-12.gh-issue-119113.kEv1Ll.rst | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-17-17-32-12.gh-issue-119113.kEv1Ll.rst diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 568a17df26fc33..3cdbb735096edb 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -205,15 +205,13 @@ def with_suffix(self, suffix): string, remove the suffix from the path. """ stem = self.stem - if not suffix: - return self.with_name(stem) - elif not stem: + if not stem: # If the stem is empty, we can't make the suffix non-empty. raise ValueError(f"{self!r} has an empty name") - elif suffix.startswith('.') and len(suffix) > 1: - return self.with_name(stem + suffix) - else: + elif suffix and not (suffix.startswith('.') and len(suffix) > 1): raise ValueError(f"Invalid suffix {suffix!r}") + else: + return self.with_name(stem + suffix) def relative_to(self, other, *, walk_up=False): """Return the relative path to another path identified by the passed diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index aadecbc142cca6..d9e51c0e3d6411 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -999,6 +999,7 @@ def test_with_suffix_windows(self): self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') + self.assertRaises(TypeError, P('c:a/b').with_suffix, None) def test_with_suffix_empty(self): P = self.cls @@ -1006,7 +1007,7 @@ def test_with_suffix_empty(self): self.assertRaises(ValueError, P('').with_suffix, '.gz') self.assertRaises(ValueError, P('/').with_suffix, '.gz') - def test_with_suffix_seps(self): + def test_with_suffix_invalid(self): P = self.cls # Invalid suffix. self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') @@ -1017,6 +1018,7 @@ def test_with_suffix_seps(self): self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') self.assertRaises(ValueError, P('a/b').with_suffix, './.d') self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') + self.assertRaises(TypeError, P('a/b').with_suffix, None) def test_relative_to_common(self): P = self.cls diff --git a/Misc/NEWS.d/next/Library/2024-05-17-17-32-12.gh-issue-119113.kEv1Ll.rst b/Misc/NEWS.d/next/Library/2024-05-17-17-32-12.gh-issue-119113.kEv1Ll.rst new file mode 100644 index 00000000000000..195be067138b2e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-17-17-32-12.gh-issue-119113.kEv1Ll.rst @@ -0,0 +1,2 @@ +Fix issue where :meth:`pathlib.PurePath.with_suffix` didn't raise +:exc:`TypeError` when given ``None`` as a suffix. From 0abf997e75bd3a8b76d920d33cc64d5e6c2d380f Mon Sep 17 00:00:00 2001 From: pulkin Date: Sun, 19 May 2024 23:46:37 +0200 Subject: [PATCH 106/903] gh-119105: difflib: improve recursion for degenerate cases (#119131) Code from https://github.com/pulkin, in PR https://github.com/python/cpython/pull/119131 Greatly speeds `Differ` when there are many identically scoring pairs, by splitting the recursion near the inputs' midpoints instead of degenerating (as now) into just peeling off the first two lines. Co-authored-by: Tim Peters --- Lib/difflib.py | 24 +++++++++++++++---- ...-05-19-12-25-36.gh-issue-119105.VcR4ig.rst | 1 + 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-19-12-25-36.gh-issue-119105.VcR4ig.rst diff --git a/Lib/difflib.py b/Lib/difflib.py index ba0b256969ebff..54ca33d5615f8d 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -911,16 +911,26 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): # don't synch up unless the lines have a similarity score of at # least cutoff; best_ratio tracks the best score seen so far - best_ratio, cutoff = 0.74, 0.75 + # best_ratio is a tuple storing the best .ratio() seen so far, and + # a measure of how far the indices are from their index range + # midpoints. The latter is used to resolve ratio ties. Favoring + # indices near the midpoints tends to cut the ranges in half. Else, + # if there are many pairs with the best ratio, recursion can grow + # very deep, and runtime becomes cubic. See: + # https://github.com/python/cpython/issues/119105 + best_ratio, cutoff = (0.74, 0), 0.75 cruncher = SequenceMatcher(self.charjunk) eqi, eqj = None, None # 1st indices of equal lines (if any) # search for the pair that matches best without being identical # (identical lines must be junk lines, & we don't want to synch up # on junk -- unless we have to) + amid = (alo + ahi - 1) / 2 + bmid = (blo + bhi - 1) / 2 for j in range(blo, bhi): bj = b[j] cruncher.set_seq2(bj) + weight_j = - abs(j - bmid) for i in range(alo, ahi): ai = a[i] if ai == bj: @@ -928,16 +938,20 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): eqi, eqj = i, j continue cruncher.set_seq1(ai) + # weight is used to balance the recursion by prioritizing + # i and j in the middle of their ranges + weight = weight_j - abs(i - amid) # computing similarity is expensive, so use the quick # upper bounds first -- have seen this speed up messy # compares by a factor of 3. # note that ratio() is only expensive to compute the first # time it's called on a sequence pair; the expensive part # of the computation is cached by cruncher - if cruncher.real_quick_ratio() > best_ratio and \ - cruncher.quick_ratio() > best_ratio and \ - cruncher.ratio() > best_ratio: - best_ratio, best_i, best_j = cruncher.ratio(), i, j + if (cruncher.real_quick_ratio(), weight) > best_ratio and \ + (cruncher.quick_ratio(), weight) > best_ratio and \ + (cruncher.ratio(), weight) > best_ratio: + best_ratio, best_i, best_j = (cruncher.ratio(), weight), i, j + best_ratio, _ = best_ratio if best_ratio < cutoff: # no non-identical "pretty close" pair if eqi is None: diff --git a/Misc/NEWS.d/next/Library/2024-05-19-12-25-36.gh-issue-119105.VcR4ig.rst b/Misc/NEWS.d/next/Library/2024-05-19-12-25-36.gh-issue-119105.VcR4ig.rst new file mode 100644 index 00000000000000..30b5f97b8059f9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-19-12-25-36.gh-issue-119105.VcR4ig.rst @@ -0,0 +1 @@ +``difflib.Differ`` is much faster for some cases of diffs where many pairs of lines are equally similar. From 357f5a1f73684d0c126a5e8f79d76ff3641c4d52 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 20 May 2024 02:04:50 -0400 Subject: [PATCH 107/903] IDLE: fix url in config.py comment (#119198) --- Lib/idlelib/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 92992fd9cce9cd..7fc08ef9748182 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -159,7 +159,7 @@ def __init__(self, _utest=False): self.userCfg = {} self.cfg = {} # TODO use to select userCfg vs defaultCfg # self.blink_off_time = ['insertofftime'] - # See https:/bugs.python.org/issue4630, msg356516. + # See https://bugs.python.org/issue4630#msg356516. if not _utest: self.CreateConfigHandlers() From 16b46ebd2b0025aa461fdfc95fbf98a4f04b49e6 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 20 May 2024 14:06:50 +0300 Subject: [PATCH 108/903] gh-119121: Fix and test `async.staggered.staggered_race` (#119173) --- Lib/asyncio/staggered.py | 3 +- Lib/test/test_asyncio/test_staggered.py | 97 +++++++++++++++++++ ...-05-19-13-05-59.gh-issue-119121.P1gnh1.rst | 2 + 3 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 Lib/test/test_asyncio/test_staggered.py create mode 100644 Misc/NEWS.d/next/Library/2024-05-19-13-05-59.gh-issue-119121.P1gnh1.rst diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index e180cde0243b15..c3a7441a7b091d 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -69,8 +69,7 @@ async def staggered_race(coro_fns, delay, *, loop=None): exceptions = [] running_tasks = [] - async def run_one_coro( - previous_failed: typing.Optional[locks.Event]) -> None: + async def run_one_coro(previous_failed) -> None: # Wait for the previous task to finish, or for delay seconds if previous_failed is not None: with contextlib.suppress(exceptions_mod.TimeoutError): diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py new file mode 100644 index 00000000000000..e6e32f7dbbbcba --- /dev/null +++ b/Lib/test/test_asyncio/test_staggered.py @@ -0,0 +1,97 @@ +import asyncio +import unittest +from asyncio.staggered import staggered_race + +from test import support + +support.requires_working_socket(module=True) + + +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + +class StaggeredTests(unittest.IsolatedAsyncioTestCase): + async def test_empty(self): + winner, index, excs = await staggered_race( + [], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(excs, []) + + async def test_one_successful(self): + async def coro(index): + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 0') + self.assertEqual(index, 0) + self.assertEqual(excs, [None]) + + async def test_first_error_second_successful(self): + async def coro(index): + if index == 0: + raise ValueError(index) + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIs(excs[1], None) + + async def test_first_timeout_second_successful(self): + async def coro(index): + if index == 0: + await asyncio.sleep(10) # much bigger than delay + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=0.1, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIs(excs[1], None) + + async def test_none_successful(self): + async def coro(index): + raise ValueError(index) + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIsInstance(excs[1], ValueError) diff --git a/Misc/NEWS.d/next/Library/2024-05-19-13-05-59.gh-issue-119121.P1gnh1.rst b/Misc/NEWS.d/next/Library/2024-05-19-13-05-59.gh-issue-119121.P1gnh1.rst new file mode 100644 index 00000000000000..fd562ea4f73317 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-19-13-05-59.gh-issue-119121.P1gnh1.rst @@ -0,0 +1,2 @@ +Fix a NameError happening in ``asyncio.staggered.staggered_race``. This +function is now tested. From 0883fd22e6d4a3e360b48f30f6afa34553b3786a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 20 May 2024 08:52:32 -0400 Subject: [PATCH 109/903] Enable some stricter mypy settings on `Lib/_pyrepl` (#119077) --- Lib/_pyrepl/mypy.ini | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini index ecd03094dbf538..395f5945ab740b 100644 --- a/Lib/_pyrepl/mypy.ini +++ b/Lib/_pyrepl/mypy.ini @@ -10,18 +10,15 @@ platform = linux pretty = True # Enable most stricter settings -enable_error_code = ignore-without-code +enable_error_code = ignore-without-code,redundant-expr strict = True # Various stricter settings that we can't yet enable # Try to enable these in the following order: -disallow_any_generics = False disallow_untyped_calls = False disallow_untyped_defs = False check_untyped_defs = False -disable_error_code = return - # Various internal modules that typeshed deliberately doesn't have stubs for: [mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] ignore_missing_imports = True From af359cee75e4806650f2b9b948e398d89ceb9555 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 20 May 2024 09:44:42 -0400 Subject: [PATCH 110/903] gh-118928: sqlite3: correctly bail if sequences of params are used with named placeholders (#119197) --- .../next/Library/2024-05-19-23-09-36.gh-issue-118928.SznMX1.rst | 2 ++ Modules/_sqlite/cursor.c | 1 + 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-05-19-23-09-36.gh-issue-118928.SznMX1.rst diff --git a/Misc/NEWS.d/next/Library/2024-05-19-23-09-36.gh-issue-118928.SznMX1.rst b/Misc/NEWS.d/next/Library/2024-05-19-23-09-36.gh-issue-118928.SznMX1.rst new file mode 100644 index 00000000000000..61b192761731d0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-19-23-09-36.gh-issue-118928.SznMX1.rst @@ -0,0 +1,2 @@ +Fix an error where incorrect bindings in :mod:`sqlite3` queries could lead +to a crash. Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 5d4b77b1a07e08..0fbd408f18cf6a 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -675,6 +675,7 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, "supplied a sequence which requires nameless (qmark) " "placeholders.", i+1, name); + return; } if (PyTuple_CheckExact(parameters)) { From 1db4695644388817c727db80cbbd38714cc5125d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 20 May 2024 09:51:02 -0400 Subject: [PATCH 111/903] gh-118928: Remove unneeded sqlite3 NEWS entry (#119208) The regression in d8e0e0091 was never part of an official release. --- .../next/Library/2024-05-19-23-09-36.gh-issue-118928.SznMX1.rst | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2024-05-19-23-09-36.gh-issue-118928.SznMX1.rst diff --git a/Misc/NEWS.d/next/Library/2024-05-19-23-09-36.gh-issue-118928.SznMX1.rst b/Misc/NEWS.d/next/Library/2024-05-19-23-09-36.gh-issue-118928.SznMX1.rst deleted file mode 100644 index 61b192761731d0..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-05-19-23-09-36.gh-issue-118928.SznMX1.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an error where incorrect bindings in :mod:`sqlite3` queries could lead -to a crash. Patch by Erlend E. Aasland. From bbb49888a752869ae93423c42039a3a8dfab34d4 Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Mon, 20 May 2024 10:28:36 -0400 Subject: [PATCH 112/903] gh-103134: Update multiprocessing.managers.ListProxy and DictProxy (GH-103133) --- Lib/multiprocessing/managers.py | 18 +++++--- Lib/test/_test_multiprocessing.py | 46 +++++++++++++++++-- ...-03-30-18-19-53.gh-issue-103134.bHrn91.rst | 6 +++ 3 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 76b915de74d94e..0f5f9f64c2de9e 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1152,10 +1152,10 @@ def set(self, value): BaseListProxy = MakeProxyType('BaseListProxy', ( - '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', - '__mul__', '__reversed__', '__rmul__', '__setitem__', - 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', - 'reverse', 'sort', '__imul__' + '__add__', '__contains__', '__delitem__', '__getitem__', '__imul__', + '__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__', + 'append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop', + 'remove', 'reverse', 'sort', )) class ListProxy(BaseListProxy): def __iadd__(self, value): @@ -1169,16 +1169,20 @@ def __imul__(self, value): _BaseDictProxy = MakeProxyType('DictProxy', ( - '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', - '__setitem__', 'clear', 'copy', 'get', 'items', + '__contains__', '__delitem__', '__getitem__', '__ior__', '__iter__', + '__len__', '__or__', '__reversed__', '__ror__', + '__setitem__', 'clear', 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' )) _BaseDictProxy._method_to_typeid_ = { '__iter__': 'Iterator', } class DictProxy(_BaseDictProxy): - __class_getitem__ = classmethod(types.GenericAlias) + def __ior__(self, value): + self._callmethod('__ior__', (value,)) + return self + __class_getitem__ = classmethod(types.GenericAlias) ArrayProxy = MakeProxyType('ArrayProxy', ( '__len__', '__getitem__', '__setitem__' diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index f126b6745dc83b..301541a666e140 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6064,12 +6064,30 @@ def _test_list(cls, obj): case.assertEqual(obj[0], 5) case.assertEqual(obj.count(5), 1) case.assertEqual(obj.index(5), 0) + obj += [7] + case.assertIsInstance(obj, multiprocessing.managers.ListProxy) + case.assertListEqual(list(obj), [5, 7]) + obj *= 2 + case.assertIsInstance(obj, multiprocessing.managers.ListProxy) + case.assertListEqual(list(obj), [5, 7, 5, 7]) + double_obj = obj * 2 + case.assertIsInstance(double_obj, list) + case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7]) + double_obj = 2 * obj + case.assertIsInstance(double_obj, list) + case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7]) + copied_obj = obj.copy() + case.assertIsInstance(copied_obj, list) + case.assertListEqual(list(copied_obj), [5, 7, 5, 7]) + obj.extend(double_obj + copied_obj) obj.sort() obj.reverse() for x in obj: pass - case.assertEqual(len(obj), 1) - case.assertEqual(obj.pop(0), 5) + case.assertEqual(len(obj), 16) + case.assertEqual(obj.pop(0), 7) + obj.clear() + case.assertEqual(len(obj), 0) def test_list(self): o = self.manager.list() @@ -6088,7 +6106,29 @@ def _test_dict(cls, obj): case.assertListEqual(list(obj.keys()), ['foo']) case.assertListEqual(list(obj.values()), [5]) case.assertDictEqual(obj.copy(), {'foo': 5}) - case.assertTupleEqual(obj.popitem(), ('foo', 5)) + obj |= {'bar': 6} + case.assertIsInstance(obj, multiprocessing.managers.DictProxy) + case.assertDictEqual(dict(obj), {'foo': 5, 'bar': 6}) + x = reversed(obj) + case.assertIsInstance(x, type(iter([]))) + case.assertListEqual(list(x), ['bar', 'foo']) + x = {'bar': 7, 'baz': 7} | obj + case.assertIsInstance(x, dict) + case.assertDictEqual(dict(x), {'foo': 5, 'bar': 6, 'baz': 7}) + x = obj | {'bar': 7, 'baz': 7} + case.assertIsInstance(x, dict) + case.assertDictEqual(dict(x), {'foo': 5, 'bar': 7, 'baz': 7}) + x = obj.fromkeys(['bar'], 6) + case.assertIsInstance(x, dict) + case.assertDictEqual(x, {'bar': 6}) + x = obj.popitem() + case.assertIsInstance(x, tuple) + case.assertTupleEqual(x, ('bar', 6)) + obj.setdefault('bar', 0) + obj.update({'bar': 7}) + case.assertEqual(obj.pop('bar'), 7) + obj.clear() + case.assertEqual(len(obj), 0) def test_dict(self): o = self.manager.dict() diff --git a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst new file mode 100644 index 00000000000000..11559dce0ae2b4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst @@ -0,0 +1,6 @@ +Add additional methods to :ref:`proxy objects ` +in the :mod:`!multiprocessing` module: + +* :meth:`!clear` and :meth:`!copy` for proxies of :class:`list` +* :meth:`~dict.fromkeys`, ``reversed(d)``, ``d | {}``, ``{} | d``, + ``d |= {'b': 2}`` for proxies of :class:`dict` From 05e1dce76d7669e90ab73e7e682360d83b8a0d02 Mon Sep 17 00:00:00 2001 From: Thanos <111999343+Sachaa-Thanasius@users.noreply.github.com> Date: Mon, 20 May 2024 10:31:43 -0400 Subject: [PATCH 113/903] gh-119185: Fix typo in `_pyrepl.pager`: `tempfilepager` should be `tempfile_pager` (#118881) Fix typo in `_pyrepl.pager`: `tempfilepager` should be `tempfile_pager` The name with no underscore doesn't exist. --- Lib/_pyrepl/pager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index 6a076b5181d872..1ac733ed3573a4 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -35,7 +35,7 @@ def get_pager() -> Pager: if os.environ.get('TERM') in ('dumb', 'emacs'): return plain_pager if sys.platform == 'win32': - return lambda text, title='': tempfilepager(plain(text), 'more <') + return lambda text, title='': tempfile_pager(plain(text), 'more <') if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: return lambda text, title='': pipe_pager(text, 'less', title) From 72d07dd30bc10751fe0298915c918eb08e555a7a Mon Sep 17 00:00:00 2001 From: David Foster Date: Mon, 20 May 2024 10:32:51 -0400 Subject: [PATCH 114/903] typing docs: Fix formatting issue (#119210) --- Doc/library/typing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 5b74dd9c9f4914..2df2faeaf39bd0 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2546,7 +2546,7 @@ types. ``__required_keys__`` and ``__optional_keys__`` rely on may not work properly, and the values of the attributes may be incorrect. - Support for :data:`ReadOnly` is reflected in the following attributes:: + Support for :data:`ReadOnly` is reflected in the following attributes: .. attribute:: __readonly_keys__ From 642b25b9a82c368b045709f0b94e7d5a02400ed2 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 20 May 2024 10:58:08 -0400 Subject: [PATCH 115/903] gh-115119: Fall back to bundled libmpdec if system libmpdec is not found (#119196) --- ...-05-19-22-54-55.gh-issue-115119.DwMwev.rst | 1 + configure | 20 +++++++++++++------ configure.ac | 19 ++++++++++++------ 3 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst new file mode 100644 index 00000000000000..acaca9e0ebbdfb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst @@ -0,0 +1 @@ +Fall back to the bundled libmpdec if a system version cannot be found. diff --git a/configure b/configure index c4b61fb8c1cfea..6cfe114fb2104c 100755 --- a/configure +++ b/configure @@ -14618,6 +14618,8 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 printf "%s\n" "$with_system_libmpdec" >&6; } + + if test "x$with_system_libmpdec" = xyes then : @@ -14697,8 +14699,10 @@ printf "%s\n" "yes" >&6; } fi else $as_nop LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no fi if test "x$with_system_libmpdec" = xyes @@ -14745,15 +14749,19 @@ LIBS=$save_LIBS else $as_nop - have_mpdec=yes - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} fi if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdecimal found; unable to build _decimal" >&5 -printf "%s\n" "$as_me: WARNING: no system libmpdecimal found; unable to build _decimal" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&5 +printf "%s\n" "$as_me: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&2;} + LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no fi # Disable forced inlining in debug builds, see GH-94847 diff --git a/configure.ac b/configure.ac index 0f1b977591c260..8657e09c9a7008 100644 --- a/configure.ac +++ b/configure.ac @@ -3980,6 +3980,13 @@ AC_ARG_WITH( [with_system_libmpdec="yes"]) AC_MSG_RESULT([$with_system_libmpdec]) +AC_DEFUN([USE_BUNDLED_LIBMPDEC], + [LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no]) + AS_VAR_IF( [with_system_libmpdec], [yes], [PKG_CHECK_MODULES( @@ -3987,9 +3994,7 @@ AS_VAR_IF( [LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} LIBMPDEC_INTERNAL=])], - [LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)"]) + [USE_BUNDLED_LIBMPDEC()]) AS_VAR_IF([with_system_libmpdec], [yes], [WITH_SAVE_ENV([ @@ -4006,13 +4011,15 @@ AS_VAR_IF([with_system_libmpdec], [yes], [have_mpdec=yes], [have_mpdec=no]) ])], - [AS_VAR_SET([have_mpdec], [yes]) - AC_MSG_WARN([m4_normalize([ + [AC_MSG_WARN([m4_normalize([ the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library.])])]) AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], - [AC_MSG_WARN([no system libmpdecimal found; unable to build _decimal])]) + [AC_MSG_WARN([m4_normalize([ + no system libmpdecimal found; falling back to bundled libmpdecimal + (deprecated and scheduled for removal in Python 3.15)])]) + USE_BUNDLED_LIBMPDEC()]) # Disable forced inlining in debug builds, see GH-94847 AS_VAR_IF( From e406b399f9f677cda3d48ed8d7c9d29a173f51f3 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Mon, 20 May 2024 17:31:45 +0200 Subject: [PATCH 116/903] [docs] TypeVarTuple default is keyword-only (#119215) --- Doc/library/typing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 2df2faeaf39bd0..a8068609fcfbe7 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1780,7 +1780,7 @@ without the dedicated syntax, as documented below. .. _typevartuple: -.. class:: TypeVarTuple(name, default=typing.NoDefault) +.. class:: TypeVarTuple(name, *, default=typing.NoDefault) Type variable tuple. A specialized form of :ref:`type variable ` that enables *variadic* generics. From 6f7dd0a4260254390d75838c84ccc7285a2264f0 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 20 May 2024 11:32:05 -0400 Subject: [PATCH 117/903] Amend NEWS category for gh-119196 (#119218) --- .../2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/{Core and Builtins => Build}/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst (100%) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst b/Misc/NEWS.d/next/Build/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst rename to Misc/NEWS.d/next/Build/2024-05-19-22-54-55.gh-issue-115119.DwMwev.rst From 19c11f244ebef492a5b20cd85a3b2e213e85388d Mon Sep 17 00:00:00 2001 From: Jeremy Hylton <32469542+jeremyhylton@users.noreply.github.com> Date: Mon, 20 May 2024 12:54:16 -0400 Subject: [PATCH 118/903] gh-119219: Remove two obsolete TODOs. (#119223) Remove two obsolete TODOs. --- Python/pyarena.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Python/pyarena.c b/Python/pyarena.c index ead03370d153c3..7ab370163b2b93 100644 --- a/Python/pyarena.c +++ b/Python/pyarena.c @@ -6,9 +6,6 @@ Measurements with standard library modules suggest the average allocation is about 20 bytes and that most compiles use a single block. - - TODO(jhylton): Think about a realloc API, maybe just for the last - allocation? */ #define DEFAULT_BLOCK_SIZE 8192 @@ -108,7 +105,6 @@ block_alloc(block *b, size_t size) /* If we need to allocate more memory than will fit in the default block, allocate a one-off block that is exactly the right size. */ - /* TODO(jhylton): Think about space waste at end of block */ block *newbl = block_new( size < DEFAULT_BLOCK_SIZE ? DEFAULT_BLOCK_SIZE : size); From 1195c164daab873ebf87ba8efe44fffdf47307ef Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 20 May 2024 13:27:09 -0400 Subject: [PATCH 119/903] gh-112844: Update CPE references for external dependencies (#118521) --- Tools/build/generate_sbom.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 258b58c03c6800..c08568f2e00326 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -305,7 +305,21 @@ def create_externals_sbom() -> None: # Set the versionInfo and downloadLocation fields for all packages. for package in sbom_data["packages"]: - package["versionInfo"] = externals_name_to_version[package["name"]] + package_version = externals_name_to_version[package["name"]] + + # Update the version information in all the locations. + package["versionInfo"] = package_version + for external_ref in package["externalRefs"]: + if external_ref["referenceType"] != "cpe23Type": + continue + # Version is the fifth field of a CPE. + cpe23ref = external_ref["referenceLocator"] + external_ref["referenceLocator"] = re.sub( + r"\A(cpe(?::[^:]+){4}):[^:]+:", + fr"\1:{package_version}:", + cpe23ref + ) + download_location = ( f"https://github.com/python/cpython-source-deps/archive/refs/tags/{externals_name_to_git_tag[package['name']]}.tar.gz" ) From 7e1a130b8ff1ed8b3a5f00fe0f06d3916b852216 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 20 May 2024 13:42:15 -0400 Subject: [PATCH 120/903] DOCS: Suggest always calling exec with a globals argument and no locals argument (GH-119235) Many users think they want a locals argument for various reasons but they do not understand that it makes code be treated as a class definition. They do not want their code treated as a class definition and get surprised. The reason not to pass locals specifically is that the following code raises a `NameError`: ```py exec(""" def f(): print("hi") f() def g(): f() g() """, {}, {}) ``` The reason not to leave out globals is as follows: ```py def t(): exec(""" def f(): print("hi") f() def g(): f() g() """) ``` --- Doc/library/functions.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 0c7ef67774cd05..3986ace02b561a 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -608,9 +608,13 @@ are always available. They are listed here in alphabetical order. will be used for both the global and the local variables. If *globals* and *locals* are given, they are used for the global and local variables, respectively. If provided, *locals* can be any mapping object. Remember - that at the module level, globals and locals are the same dictionary. If exec - gets two separate objects as *globals* and *locals*, the code will be - executed as if it were embedded in a class definition. + that at the module level, globals and locals are the same dictionary. + + .. note:: + + Most users should just pass a *globals* argument and never *locals*. + If exec gets two separate objects as *globals* and *locals*, the code + will be executed as if it were embedded in a class definition. If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module From c0d81b256604a1079349d82d136db43eefcb3df1 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Mon, 20 May 2024 19:21:56 +0100 Subject: [PATCH 121/903] gh-118877: Fix AssertionError crash in pyrepl (#118936) --- Lib/_pyrepl/commands.py | 4 +--- Lib/test/test_pyrepl.py | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 456cba0769c952..51c7afebede5a8 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -34,9 +34,7 @@ # types if False: - from .reader import Reader from .historical_reader import HistoricalReader - from .console import Event class Command: @@ -245,7 +243,7 @@ def do(self) -> None: x, y = r.pos2xy() new_y = y - 1 - if new_y < 0: + if r.bol() == 0: if r.historyi > 0: r.select_item(r.historyi - 1) return diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index c8990b699b214c..ee6ba658f11e39 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -607,6 +607,30 @@ def test_global_namespace_completion(self): output = multiline_input(reader, namespace) self.assertEqual(output, "python") + def test_updown_arrow_with_completion_menu(self): + """Up arrow in the middle of unfinished tab completion when the menu is displayed + should work and trigger going back in history. Down arrow should subsequently + get us back to the incomplete command.""" + code = "import os\nos.\t\t" + namespace = {"os": os} + + events = itertools.chain( + code_to_events(code), + [ + Event(evt='key', data='up', raw=bytearray(b'\x1bOA')), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + code_to_events("\n") + ) + reader = self.prepare_reader(events, namespace=namespace) + output = multiline_input(reader, namespace) + # This is the first line, nothing to see here + self.assertEqual(output, "import os") + # This is the second line. We pressed up and down arrows + # so we should end up where we were when we initiated tab completion. + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.") + @patch("_pyrepl.curses.tigetstr", lambda x: b"") class TestUnivEventQueue(TestCase): @@ -1001,6 +1025,5 @@ def test_up_arrow_after_ctrl_r(self): reader, _ = handle_all_events(events) self.assert_screen_equals(reader, "") - if __name__ == '__main__': unittest.main() From a6fdb31b6714c9f3c65fefbb3fe388b2b139a75f Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Mon, 20 May 2024 12:10:47 -0700 Subject: [PATCH 122/903] gh-92081: Fix for email.generator.Generator with whitespace between encoded words. (#92281) * Fix for email.generator.Generator with whitespace between encoded words. email.generator.Generator currently does not handle whitespace between encoded words correctly when the encoded words span multiple lines. The current generator will create an encoded word for each line. If the end of the line happens to correspond with the end real word in the plaintext, the generator will place an unencoded space at the start of the subsequent lines to represent the whitespace between the plaintext words. A compliant decoder will strip all the whitespace from between two encoded words which leads to missing spaces in the round-tripped output. The fix for this is to make sure that whitespace between two encoded words ends up inside of one or the other of the encoded words. This fix places the space inside of the second encoded word. A second problem happens with continuation lines. A continuation line that starts with whitespace and is followed by a non-encoded word is fine because the newline between such continuation lines is defined as condensing to a single space character. When the continuation line starts with whitespace followed by an encoded word, however, the RFCs specify that the word is run together with the encoded word on the previous line. This is because normal words are filded on syntactic breaks by encoded words are not. The solution to this is to add the whitespace to the start of the encoded word on the continuation line. Test cases are from #92081 * Rename a variable so it's not confused with the final variable. --- Lib/email/_header_value_parser.py | 48 ++++++++++++++++--- Lib/test/test_email/test_generator.py | 35 ++++++++++++++ Lib/test/test_email/test_headerregistry.py | 3 +- ...3-04-26-22-24-17.gh-issue-92081.V8xMot.rst | 1 + 4 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-26-22-24-17.gh-issue-92081.V8xMot.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index d1b4c7df4f445f..6148801460cfb2 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -2784,11 +2784,15 @@ def _refold_parse_tree(parse_tree, *, policy): # max_line_length 0/None means no limit, ie: infinitely long. maxlen = policy.max_line_length or sys.maxsize encoding = 'utf-8' if policy.utf8 else 'us-ascii' - lines = [''] - last_ew = None + lines = [''] # Folded lines to be output + leading_whitespace = '' # When we have whitespace between two encoded + # words, we may need to encode the whitespace + # at the beginning of the second word. + last_ew = None # Points to the last encoded character if there's an ew on + # the line last_charset = None wrap_as_ew_blocked = 0 - want_encoding = False + want_encoding = False # This is set to True if we need to encode this part end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') parts = list(parse_tree) while parts: @@ -2812,10 +2816,12 @@ def _refold_parse_tree(parse_tree, *, policy): # 'charset' property on the policy. charset = 'utf-8' want_encoding = True + if part.token_type == 'mime-parameters': # Mime parameter folding (using RFC2231) is extra special. _fold_mime_parameters(part, lines, maxlen, encoding) continue + if want_encoding and not wrap_as_ew_blocked: if not part.as_ew_allowed: want_encoding = False @@ -2847,21 +2853,38 @@ def _refold_parse_tree(parse_tree, *, policy): last_charset == 'utf-8' and charset != 'us-ascii')): last_ew = None last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, - part.ew_combine_allowed, charset) + part.ew_combine_allowed, charset, leading_whitespace) + # This whitespace has been added to the lines in _fold_as_ew() + # so clear it now. + leading_whitespace = '' last_charset = charset want_encoding = False continue + if len(tstr) <= maxlen - len(lines[-1]): lines[-1] += tstr continue + # This part is too long to fit. The RFC wants us to break at # "major syntactic breaks", so unless we don't consider this # to be one, check if it will fit on the next line by itself. + leading_whitespace = '' if (part.syntactic_break and len(tstr) + 1 <= maxlen): newline = _steal_trailing_WSP_if_exists(lines) if newline or part.startswith_fws(): + # We're going to fold the data onto a new line here. Due to + # the way encoded strings handle continuation lines, we need to + # be prepared to encode any whitespace if the next line turns + # out to start with an encoded word. lines.append(newline + tstr) + + whitespace_accumulator = [] + for char in lines[-1]: + if char not in WSP: + break + whitespace_accumulator.append(char) + leading_whitespace = ''.join(whitespace_accumulator) last_ew = None continue if not hasattr(part, 'encode'): @@ -2885,9 +2908,10 @@ def _refold_parse_tree(parse_tree, *, policy): else: # We can't fold it onto the next line either... lines[-1] += tstr + return policy.linesep.join(lines) + policy.linesep -def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): +def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, leading_whitespace): """Fold string to_encode into lines as encoded word, combining if allowed. Return the new value for last_ew, or None if ew_combine_allowed is False. @@ -2902,7 +2926,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): to_encode = str( get_unstructured(lines[-1][last_ew:] + to_encode)) lines[-1] = lines[-1][:last_ew] - if to_encode[0] in WSP: + elif to_encode[0] in WSP: # We're joining this to non-encoded text, so don't encode # the leading blank. leading_wsp = to_encode[0] @@ -2910,6 +2934,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): if (len(lines[-1]) == maxlen): lines.append(_steal_trailing_WSP_if_exists(lines)) lines[-1] += leading_wsp + trailing_wsp = '' if to_encode[-1] in WSP: # Likewise for the trailing space. @@ -2929,11 +2954,20 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): while to_encode: remaining_space = maxlen - len(lines[-1]) - text_space = remaining_space - chrome_len + text_space = remaining_space - chrome_len - len(leading_whitespace) if text_space <= 0: lines.append(' ') continue + # If we are at the start of a continuation line, prepend whitespace + # (we only want to do this when the line starts with an encoded word + # but if we're folding in this helper function, then we know that we + # are going to be writing out an encoded word.) + if len(lines) > 1 and len(lines[-1]) == 1 and leading_whitespace: + encoded_word = _ew.encode(leading_whitespace, charset=encode_as) + lines[-1] += encoded_word + leading_whitespace = '' + to_encode_word = to_encode[:text_space] encoded_word = _ew.encode(to_encode_word, charset=encode_as) excess = len(encoded_word) - remaining_space diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index 3ebcb684d006d0..bfff1051262079 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -281,6 +281,41 @@ class TestBytesGenerator(TestGeneratorBase, TestEmailBase): ioclass = io.BytesIO typ = lambda self, x: x.encode('ascii') + def test_defaults_handle_spaces_between_encoded_words_when_folded(self): + source = ("Уведомление о принятии в работу обращения для" + " подключения услуги") + expected = ('Subject: =?utf-8?b?0KPQstC10LTQvtC80LvQtdC90LjQtSDQviDQv9GA0LjQvdGP0YLQuNC4?=\n' + ' =?utf-8?b?INCyINGA0LDQsdC+0YLRgyDQvtCx0YDQsNGJ0LXQvdC40Y8g0LTQu9GPINC/0L4=?=\n' + ' =?utf-8?b?0LTQutC70Y7Rh9C10L3QuNGPINGD0YHQu9GD0LPQuA==?=\n\n').encode('ascii') + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_defaults_handle_spaces_at_start_of_subject(self): + source = " Уведомление" + expected = b"Subject: =?utf-8?b?0KPQstC10LTQvtC80LvQtdC90LjQtQ==?=\n\n" + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_defaults_handle_spaces_at_start_of_continuation_line(self): + source = " ф ффффффффффффффффффф ф ф" + expected = (b"Subject: " + b"=?utf-8?b?0YQg0YTRhNGE0YTRhNGE0YTRhNGE0YTRhNGE0YTRhNGE0YTRhNGE0YQ=?=\n" + b" =?utf-8?b?INGEINGE?=\n\n") + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + def test_cte_type_7bit_handles_unknown_8bit(self): source = ("Subject: Maintenant je vous présente mon " "collègue\n\n").encode('utf-8') diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index bb7ca8dfd8c52c..5a608a033c7e54 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -7,6 +7,7 @@ from test.test_email import TestEmailBase, parameterize from email import headerregistry from email.headerregistry import Address, Group +from email.header import decode_header from test.support import ALWAYS_EQ @@ -1648,7 +1649,7 @@ def test_address_display_names(self): 'Lôrem ipsum dôlôr sit amet, cônsectetuer adipiscing. ' 'Suspendisse pôtenti. Aliquam nibh. Suspendisse pôtenti.', '=?utf-8?q?L=C3=B4rem_ipsum_d=C3=B4l=C3=B4r_sit_amet=2C_c' - '=C3=B4nsectetuer?=\n =?utf-8?q?adipiscing=2E_Suspendisse' + '=C3=B4nsectetuer?=\n =?utf-8?q?_adipiscing=2E_Suspendisse' '_p=C3=B4tenti=2E_Aliquam_nibh=2E?=\n Suspendisse =?utf-8' '?q?p=C3=B4tenti=2E?=', ), diff --git a/Misc/NEWS.d/next/Library/2023-04-26-22-24-17.gh-issue-92081.V8xMot.rst b/Misc/NEWS.d/next/Library/2023-04-26-22-24-17.gh-issue-92081.V8xMot.rst new file mode 100644 index 00000000000000..0302e957b884cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-26-22-24-17.gh-issue-92081.V8xMot.rst @@ -0,0 +1 @@ +Fix missing spaces in email headers when the spaces are mixed with encoded 8-bit characters. From 6b80a5b20f31a067bd1c374295608df5f1210f49 Mon Sep 17 00:00:00 2001 From: Tialo <65392801+Tialo@users.noreply.github.com> Date: Mon, 20 May 2024 22:17:44 +0300 Subject: [PATCH 123/903] Use correct markup in unittest.mock.reset_mock documentation (GH-119207) --- Doc/library/unittest.mock.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 9496c35f5267d0..0c1cecf45381e2 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -414,13 +414,13 @@ the *new_callable* argument to :func:`patch`. This can be useful where you want to make a series of assertions that reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the - return value, :attr:`side_effect` or any child attributes you have + :attr:`return_value`, :attr:`side_effect` or any child attributes you have set using normal assignment by default. In case you want to reset - *return_value* or :attr:`side_effect`, then pass the corresponding + :attr:`return_value` or :attr:`side_effect`, then pass the corresponding parameter as ``True``. Child mocks and the return value mock (if any) are reset as well. - .. note:: *return_value*, and :attr:`side_effect` are keyword-only + .. note:: *return_value*, and *side_effect* are keyword-only arguments. From e188527c343c74574d481b77c30063db1436e62b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 20 May 2024 22:21:04 +0300 Subject: [PATCH 124/903] gh-118760: Restore the default value of tkinter.wantobjects to 1 (GH-118784) It was set to 2 in 65f5e586a1239ed1a66d8284773d7b02ce40e480 (GH-98592). --- Doc/whatsnew/3.13.rst | 10 ---------- Lib/tkinter/__init__.py | 2 +- Misc/NEWS.d/3.13.0b1.rst | 14 +++++++------- .../2024-05-08-21-30-33.gh-issue-118760.XvyMHn.rst | 1 + 4 files changed, 9 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-08-21-30-33.gh-issue-118760.XvyMHn.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index effa554bfe8469..dd5190d63ec579 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -2177,16 +2177,6 @@ Changes in the Python API returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. (Contributed by Serhiy Storchaka in :gh:`115961`.) -* Callbacks registered in the :mod:`tkinter` module now take arguments as - various Python objects (``int``, ``float``, ``bytes``, ``tuple``), - not just ``str``. - To restore the previous behavior set :mod:`!tkinter` module global - :data:`!wantobject` to ``1`` before creating the - :class:`!Tk` object or call the :meth:`!wantobject` - method of the :class:`!Tk` object with argument ``1``. - Calling it with argument ``2`` restores the current default behavior. - (Contributed by Serhiy Storchaka in :gh:`66410`.) - Changes in the C API -------------------- diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index f03da0ff5f98ec..5352276e874bf5 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -40,7 +40,7 @@ from tkinter.constants import * import re -wantobjects = 2 +wantobjects = 1 _debug = False # set to True to print executed Tcl/Tk commands TkVersion = float(_tkinter.TK_VERSION) diff --git a/Misc/NEWS.d/3.13.0b1.rst b/Misc/NEWS.d/3.13.0b1.rst index 8d49ff06efd07b..c3bfcc83f2ae7c 100644 --- a/Misc/NEWS.d/3.13.0b1.rst +++ b/Misc/NEWS.d/3.13.0b1.rst @@ -1346,13 +1346,13 @@ urllib. .. nonce: du4UKW .. section: Library -Callbacks registered in the :mod:`tkinter` module now take arguments as -various Python objects (``int``, ``float``, ``bytes``, ``tuple``), not just -``str``. To restore the previous behavior set :mod:`!tkinter` module global -:data:`~tkinter.wantobject` to ``1`` before creating the -:class:`~tkinter.Tk` object or call the :meth:`~tkinter.Tk.wantobject` -method of the :class:`!Tk` object with argument ``1``. Calling it with -argument ``2`` restores the current default behavior. +Setting the :mod:`!tkinter` module global :data:`~tkinter.wantobject` to ``2`` +before creating the :class:`~tkinter.Tk` object or call the +:meth:`~tkinter.Tk.wantobject` method of the :class:`!Tk` object with argument +``2`` makes now arguments to callbacks registered in the :mod:`tkinter` module +to be passed as various Python objects (``int``, ``float``, ``bytes``, ``tuple``), +depending on their internal represenation in Tcl, instead of always ``str``. +:data:`!tkinter.wantobject` is now set to ``2`` by default. .. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-21-30-33.gh-issue-118760.XvyMHn.rst b/Misc/NEWS.d/next/Library/2024-05-08-21-30-33.gh-issue-118760.XvyMHn.rst new file mode 100644 index 00000000000000..0e2712c26b1c13 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-08-21-30-33.gh-issue-118760.XvyMHn.rst @@ -0,0 +1 @@ +Restore the default value of ``tkiter.wantobjects`` to ``1``. From 034cf0c3167c850c8341deb61e210cb0dbcdb02d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 20 May 2024 15:31:38 -0400 Subject: [PATCH 125/903] Docs: Ensure no warnings are found in the NEWS file before a given line number (#119221) --- .github/workflows/reusable-docs.yml | 3 +- Doc/tools/check-warnings.py | 40 +++++++++++++++++++ ...-05-07-16-57-56.gh-issue-118561.wNMKVd.rst | 4 +- ...3-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst | 2 +- ...-05-08-19-47-34.gh-issue-101357.e4R_9x.rst | 4 +- 5 files changed, 47 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 9e26d7847d2bd3..859f78d043ba92 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -62,7 +62,8 @@ jobs: python Doc/tools/check-warnings.py \ --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ --fail-if-regression \ - --fail-if-improved + --fail-if-improved \ + --fail-if-new-news-nit # This build doesn't use problem matchers or check annotations build_doc_oldest_supported_sphinx: diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py index 809a8d63087e12..c50b00636c36ce 100644 --- a/Doc/tools/check-warnings.py +++ b/Doc/tools/check-warnings.py @@ -13,6 +13,9 @@ from pathlib import Path from typing import TextIO +# Fail if NEWS nit found before this line number +NEWS_NIT_THRESHOLD = 200 + # Exclude these whether they're dirty or clean, # because they trigger a rebuild of dirty files. EXCLUDE_FILES = { @@ -245,6 +248,32 @@ def fail_if_improved( return 0 +def fail_if_new_news_nit(warnings: list[str], threshold: int) -> int: + """ + Ensure no warnings are found in the NEWS file before a given line number. + """ + news_nits = ( + warning + for warning in warnings + if "/build/NEWS:" in warning + ) + + # Nits found before the threshold line + new_news_nits = [ + nit + for nit in news_nits + if int(nit.split(":")[1]) <= threshold + ] + + if new_news_nits: + print("\nError: new NEWS nits:\n") + for warning in new_news_nits: + print(warning) + return -1 + + return 0 + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( @@ -264,6 +293,14 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Fail if new files with no nits are found", ) + parser.add_argument( + "--fail-if-new-news-nit", + metavar="threshold", + type=int, + nargs="?", + const=NEWS_NIT_THRESHOLD, + help="Fail if new NEWS nit found before threshold line number", + ) args = parser.parse_args(argv) if args.annotate_diff is not None and len(args.annotate_diff) > 2: @@ -304,6 +341,9 @@ def main(argv: list[str] | None = None) -> int: if args.fail_if_improved: exit_code += fail_if_improved(files_with_expected_nits, files_with_nits) + if args.fail_if_new_news_nit: + exit_code += fail_if_new_news_nit(warnings, args.fail_if_new_news_nit) + return exit_code diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst index 9eaf0abb8a6128..c506a8cefd00f1 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst @@ -1,2 +1,2 @@ -Fix race condition in free-threaded build where :meth:`list.extend` could expose -uninitialied memory to concurrent readers. +Fix race condition in free-threaded build where :meth:`!list.extend` could +expose uninitialised memory to concurrent readers. diff --git a/Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst b/Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst index 6f13188f6f119f..2c736e72476313 100644 --- a/Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst +++ b/Misc/NEWS.d/next/Library/2023-04-10-00-04-37.gh-issue-87106.UyBnPQ.rst @@ -1,3 +1,3 @@ -Fixed handling in :meth:`inspect.signature.bind` of keyword arguments having +Fixed handling in :meth:`inspect.Signature.bind` of keyword arguments having the same name as positional-only arguments when a variadic keyword argument (e.g. ``**kwargs``) is present. diff --git a/Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst b/Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst index 9fad7a416fcc24..c99a7e5f024823 100644 --- a/Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst +++ b/Misc/NEWS.d/next/Library/2024-05-08-19-47-34.gh-issue-101357.e4R_9x.rst @@ -1,5 +1,5 @@ Suppress all :exc:`OSError` exceptions from :meth:`pathlib.Path.exists` and ``is_*()`` methods, rather than a selection of more common errors. The new behaviour is consistent with :func:`os.path.exists`, :func:`os.path.isdir`, -etc. Use :meth:`Path.stat` to retrieve the file status without suppressing -exceptions. +etc. Use :meth:`pathlib.Path.stat` to retrieve the file status without +suppressing exceptions. From fe67af19638d208239549ccac8b4f4fb6480e801 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 20 May 2024 22:34:48 +0300 Subject: [PATCH 126/903] gh-119189: Add more tests for mixed Fraction arithmetic (GH-119236) --- Lib/test/test_fractions.py | 263 +++++++++++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index b45bd098a36684..3a9a86fe7a8b67 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1,5 +1,6 @@ """Tests for Lib/fractions.py.""" +import cmath from decimal import Decimal from test.support import requires_IEEE_754 import math @@ -91,6 +92,187 @@ class DummyFraction(fractions.Fraction): def _components(r): return (r.numerator, r.denominator) +def typed_approx_eq(a, b): + return type(a) == type(b) and (a == b or math.isclose(a, b)) + +class Symbolic: + """Simple non-numeric class for testing mixed arithmetic. + It is not Integral, Rational, Real or Complex, and cannot be conveted + to int, float or complex. but it supports some arithmetic operations. + """ + def __init__(self, value): + self.value = value + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} * {other}') + def __rmul__(self, other): + return self.__class__(f'{other} * {self}') + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} / {other}') + def __rtruediv__(self, other): + return self.__class__(f'{other} / {self}') + def __mod__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} % {other}') + def __rmod__(self, other): + return self.__class__(f'{other} % {self}') + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} ** {other}') + def __rpow__(self, other): + return self.__class__(f'{other} ** {self}') + def __eq__(self, other): + if other.__class__ != self.__class__: + return NotImplemented + return self.value == other.value + def __str__(self): + return f'{self.value}' + def __repr__(self): + return f'{self.__class__.__name__}({self.value!r})' + +class Rat: + """Simple Rational class for testing mixed arithmetic.""" + def __init__(self, n, d): + self.numerator = n + self.denominator = d + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator * other.numerator, + self.denominator * other.denominator) + def __rmul__(self, other): + return self.__class__(other.numerator * self.numerator, + other.denominator * self.denominator) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator * other.denominator, + self.denominator * other.numerator) + def __rtruediv__(self, other): + return self.__class__(other.numerator * self.denominator, + other.denominator * self.numerator) + def __mod__(self, other): + if isinstance(other, F): + return NotImplemented + d = self.denominator * other.numerator + return self.__class__(self.numerator * other.denominator % d, d) + def __rmod__(self, other): + d = other.denominator * self.numerator + return self.__class__(other.numerator * self.denominator % d, d) + + return self.__class__(other.numerator / self.numerator, + other.denominator / self.denominator) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator ** other, + self.denominator ** other) + def __float__(self): + return self.numerator / self.denominator + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return (typed_approx_eq(self.numerator, other.numerator) and + typed_approx_eq(self.denominator, other.denominator)) + def __repr__(self): + return f'{self.__class__.__name__}({self.numerator!r}, {self.denominator!r})' +numbers.Rational.register(Rat) + +class Root: + """Simple Real class for testing mixed arithmetic.""" + def __init__(self, v, n=F(2)): + self.base = v + self.degree = n + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base * other**self.degree, self.degree) + def __rmul__(self, other): + return self.__class__(other**self.degree * self.base, self.degree) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base / other**self.degree, self.degree) + def __rtruediv__(self, other): + return self.__class__(other**self.degree / self.base, self.degree) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base, self.degree / other) + def __float__(self): + return float(self.base) ** (1 / float(self.degree)) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.base, other.base) and typed_approx_eq(self.degree, other.degree) + def __repr__(self): + return f'{self.__class__.__name__}({self.base!r}, {self.degree!r})' +numbers.Real.register(Root) + +class Polar: + """Simple Complex class for testing mixed arithmetic.""" + def __init__(self, r, phi): + self.r = r + self.phi = phi + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r * other, self.phi) + def __rmul__(self, other): + return self.__class__(other * self.r, self.phi) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r / other, self.phi) + def __rtruediv__(self, other): + return self.__class__(other / self.r, -self.phi) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r ** other, self.phi * other) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.r, other.r) and typed_approx_eq(self.phi, other.phi) + def __repr__(self): + return f'{self.__class__.__name__}({self.r!r}, {self.phi!r})' +numbers.Complex.register(Polar) + +class Rect: + """Other simple Complex class for testing mixed arithmetic.""" + def __init__(self, x, y): + self.x = x + self.y = y + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.x * other, self.y * other) + def __rmul__(self, other): + return self.__class__(other * self.x, other * self.y) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.x / other, self.y / other) + def __rtruediv__(self, other): + r = self.x * self.x + self.y * self.y + return self.__class__(other * (self.x / r), other * (self.y / r)) + def __rpow__(self, other): + return Polar(other ** self.x, math.log(other) * self.y) + def __complex__(self): + return complex(self.x, self.y) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.x, other.x) and typed_approx_eq(self.y, other.y) + def __repr__(self): + return f'{self.__class__.__name__}({self.x!r}, {self.y!r})' +numbers.Complex.register(Rect) + class FractionTest(unittest.TestCase): @@ -593,6 +775,7 @@ def testMixedArithmetic(self): self.assertTypedEquals(0.9, 1.0 - F(1, 10)) self.assertTypedEquals(0.9 + 0j, (1.0 + 0j) - F(1, 10)) + def testMixedMultiplication(self): self.assertTypedEquals(F(1, 10), F(1, 10) * 1) self.assertTypedEquals(0.1, F(1, 10) * 1.0) self.assertTypedEquals(0.1 + 0j, F(1, 10) * (1.0 + 0j)) @@ -600,6 +783,24 @@ def testMixedArithmetic(self): self.assertTypedEquals(0.1, 1.0 * F(1, 10)) self.assertTypedEquals(0.1 + 0j, (1.0 + 0j) * F(1, 10)) + self.assertTypedEquals(F(3, 2) * DummyFraction(5, 3), F(5, 2)) + self.assertTypedEquals(DummyFraction(5, 3) * F(3, 2), F(5, 2)) + self.assertTypedEquals(F(3, 2) * Rat(5, 3), Rat(15, 6)) + self.assertTypedEquals(Rat(5, 3) * F(3, 2), F(5, 2)) + + self.assertTypedEquals(F(3, 2) * Root(4), Root(F(9, 1))) + self.assertTypedEquals(Root(4) * F(3, 2), 3.0) + + self.assertTypedEquals(F(3, 2) * Polar(4, 2), Polar(F(6, 1), 2)) + self.assertTypedEquals(F(3, 2) * Polar(4.0, 2), Polar(6.0, 2)) + self.assertTypedEquals(F(3, 2) * Rect(4, 3), Rect(F(6, 1), F(9, 2))) + self.assertRaises(TypeError, operator.mul, Polar(4, 2), F(3, 2)) + self.assertTypedEquals(Rect(4, 3) * F(3, 2), 6.0 + 4.5j) + + self.assertEqual(F(3, 2) * Symbolic('X'), Symbolic('3/2 * X')) + self.assertRaises(TypeError, operator.mul, Symbolic('X'), F(3, 2)) + + def testMixedDivision(self): self.assertTypedEquals(F(1, 10), F(1, 10) / 1) self.assertTypedEquals(0.1, F(1, 10) / 1.0) self.assertTypedEquals(0.1 + 0j, F(1, 10) / (1.0 + 0j)) @@ -607,6 +808,24 @@ def testMixedArithmetic(self): self.assertTypedEquals(10.0, 1.0 / F(1, 10)) self.assertTypedEquals(10.0 + 0j, (1.0 + 0j) / F(1, 10)) + self.assertTypedEquals(F(3, 2) / DummyFraction(3, 5), F(5, 2)) + self.assertTypedEquals(DummyFraction(5, 3) / F(2, 3), F(5, 2)) + self.assertTypedEquals(F(3, 2) / Rat(3, 5), Rat(15, 6)) + self.assertTypedEquals(Rat(5, 3) / F(2, 3), F(5, 2)) + + self.assertTypedEquals(F(2, 3) / Root(4), Root(F(1, 9))) + self.assertTypedEquals(Root(4) / F(2, 3), 3.0) + + self.assertTypedEquals(F(3, 2) / Polar(4, 2), Polar(F(3, 8), -2)) + self.assertTypedEquals(F(3, 2) / Polar(4.0, 2), Polar(0.375, -2)) + self.assertTypedEquals(F(3, 2) / Rect(4, 3), Rect(0.24, 0.18)) + self.assertRaises(TypeError, operator.truediv, Polar(4, 2), F(2, 3)) + self.assertTypedEquals(Rect(4, 3) / F(2, 3), 6.0 + 4.5j) + + self.assertEqual(F(3, 2) / Symbolic('X'), Symbolic('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, Symbolic('X'), F(2, 3)) + + def testMixedIntegerDivision(self): self.assertTypedEquals(0, F(1, 10) // 1) self.assertTypedEquals(0.0, F(1, 10) // 1.0) self.assertTypedEquals(10, 1 // F(1, 10)) @@ -631,6 +850,21 @@ def testMixedArithmetic(self): self.assertTypedTupleEquals(divmod(-0.1, float('inf')), divmod(F(-1, 10), float('inf'))) self.assertTypedTupleEquals(divmod(-0.1, float('-inf')), divmod(F(-1, 10), float('-inf'))) + self.assertTypedEquals(F(3, 2) % DummyFraction(3, 5), F(3, 10)) + self.assertTypedEquals(DummyFraction(5, 3) % F(2, 3), F(1, 3)) + self.assertTypedEquals(F(3, 2) % Rat(3, 5), Rat(3, 6)) + self.assertTypedEquals(Rat(5, 3) % F(2, 3), F(1, 3)) + + self.assertRaises(TypeError, operator.mod, F(2, 3), Root(4)) + self.assertTypedEquals(Root(4) % F(3, 2), 0.5) + + self.assertRaises(TypeError, operator.mod, F(3, 2), Polar(4, 2)) + self.assertRaises(TypeError, operator.mod, Rect(4, 3), F(2, 3)) + + self.assertEqual(F(3, 2) % Symbolic('X'), Symbolic('3/2 % X')) + self.assertRaises(TypeError, operator.mod, Symbolic('X'), F(2, 3)) + + def testMixedPower(self): # ** has more interesting conversion rules. self.assertTypedEquals(F(100, 1), F(1, 10) ** -2) self.assertTypedEquals(F(100, 1), F(10, 1) ** 2) @@ -647,6 +881,35 @@ def testMixedArithmetic(self): self.assertRaises(ZeroDivisionError, operator.pow, F(0, 1), -2) + self.assertTypedEquals(F(3, 2) ** Rat(3, 1), F(27, 8)) + self.assertTypedEquals(F(3, 2) ** Rat(-3, 1), F(8, 27)) + self.assertTypedEquals(F(-3, 2) ** Rat(-3, 1), F(-8, 27)) + self.assertTypedEquals(F(9, 4) ** Rat(3, 2), 3.375) + self.assertIsInstance(F(4, 9) ** Rat(-3, 2), float) + self.assertAlmostEqual(F(4, 9) ** Rat(-3, 2), 3.375) + self.assertAlmostEqual(F(-4, 9) ** Rat(-3, 2), 3.375j) + + self.assertTypedEquals(Rat(9, 4) ** F(3, 2), 3.375) + self.assertTypedEquals(Rat(3, 2) ** F(3, 1), Rat(27, 8)) + self.assertTypedEquals(Rat(3, 2) ** F(-3, 1), F(8, 27)) + self.assertIsInstance(Rat(4, 9) ** F(-3, 2), float) + self.assertAlmostEqual(Rat(4, 9) ** F(-3, 2), 3.375) + + self.assertTypedEquals(Root(4) ** F(2, 3), Root(4, 3.0)) + self.assertTypedEquals(Root(4) ** F(2, 1), Root(4, F(1))) + self.assertTypedEquals(Root(4) ** F(-2, 1), Root(4, -F(1))) + self.assertTypedEquals(Root(4) ** F(-2, 3), Root(4, -3.0)) + + self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(2.25, 0.0)) + self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(1.0, 0.0)) + self.assertTypedEquals(Polar(4, 2) ** F(3, 2), Polar(8.0, 3.0)) + self.assertTypedEquals(Polar(4, 2) ** F(3, 1), Polar(64, 6)) + self.assertTypedEquals(Polar(4, 2) ** F(-3, 1), Polar(0.015625, -6)) + self.assertTypedEquals(Polar(4, 2) ** F(-3, 2), Polar(0.125, -3.0)) + + self.assertTypedEquals(F(3, 2) ** Symbolic('X'), Symbolic('1.5 ** X')) + self.assertTypedEquals(Symbolic('X') ** F(3, 2), Symbolic('X ** 1.5')) + def testMixingWithDecimal(self): # Decimal refuses mixed arithmetic (but not mixed comparisons) self.assertRaises(TypeError, operator.add, From a443e542811cd8242d6f42e817b0f0af0dd2fd92 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 20 May 2024 16:04:52 -0400 Subject: [PATCH 127/903] gh-111201: Add more tests to test_pyrepl to cover key translation (#118705) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/test/test_pyrepl.py | 189 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 181 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index ee6ba658f11e39..1daf3a8b498119 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -23,7 +23,8 @@ from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig from _pyrepl.simple_interact import _strip_final_indent from _pyrepl.unix_eventqueue import EventQueue -from _pyrepl.simple_interact import InteractiveColoredConsole +from _pyrepl.input import KeymapTranslator +from _pyrepl.keymap import parse_keys, compile_keymap def more_lines(unicodetext, namespace=None): @@ -617,10 +618,10 @@ def test_updown_arrow_with_completion_menu(self): events = itertools.chain( code_to_events(code), [ - Event(evt='key', data='up', raw=bytearray(b'\x1bOA')), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), ], - code_to_events("\n") + code_to_events("\n"), ) reader = self.prepare_reader(events, namespace=namespace) output = multiline_input(reader, namespace) @@ -1017,13 +1018,185 @@ def test_setpos_fromxy_in_wrapped_line(self): self.assertEqual(reader.pos, 9) def test_up_arrow_after_ctrl_r(self): - events = iter([ - Event(evt='key', data='\x12', raw=bytearray(b'\x12')), - Event(evt='key', data='up', raw=bytearray(b'\x1bOA')), - ]) + events = iter( + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ] + ) reader, _ = handle_all_events(events) self.assert_screen_equals(reader, "") -if __name__ == '__main__': + +class KeymapTranslatorTests(unittest.TestCase): + def test_push_single_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_multiple_keys(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_invalid_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "b") + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["b"])) + + def test_push_invalid_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_character_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_character_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + evt3 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key_interrupted(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + evt3 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_invalid_key_with_unicode_category(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "\u0003") # Control character + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["\u0003"])) + + def test_empty(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + self.assertTrue(translator.empty()) + evt = Event("key", "a") + translator.push(evt) + self.assertFalse(translator.empty()) + translator.get() + self.assertTrue(translator.empty()) + + +class TestParseKeys(unittest.TestCase): + def test_single_character(self): + self.assertEqual(parse_keys("a"), ["a"]) + self.assertEqual(parse_keys("b"), ["b"]) + self.assertEqual(parse_keys("1"), ["1"]) + + def test_escape_sequences(self): + self.assertEqual(parse_keys("\\n"), ["\n"]) + self.assertEqual(parse_keys("\\t"), ["\t"]) + self.assertEqual(parse_keys("\\\\"), ["\\"]) + self.assertEqual(parse_keys("\\'"), ["'"]) + self.assertEqual(parse_keys('\\"'), ['"']) + + def test_control_sequences(self): + self.assertEqual(parse_keys("\\C-a"), ["\x01"]) + self.assertEqual(parse_keys("\\C-b"), ["\x02"]) + self.assertEqual(parse_keys("\\C-c"), ["\x03"]) + + def test_meta_sequences(self): + self.assertEqual(parse_keys("\\M-a"), ["\033", "a"]) + self.assertEqual(parse_keys("\\M-b"), ["\033", "b"]) + self.assertEqual(parse_keys("\\M-c"), ["\033", "c"]) + + def test_keynames(self): + self.assertEqual(parse_keys("\\"), ["up"]) + self.assertEqual(parse_keys("\\"), ["down"]) + self.assertEqual(parse_keys("\\"), ["left"]) + self.assertEqual(parse_keys("\\"), ["right"]) + + def test_combinations(self): + self.assertEqual(parse_keys("\\C-a\\n\\"), ["\x01", "\n", "up"]) + self.assertEqual(parse_keys("\\M-a\\t\\"), ["\033", "a", "\t", "down"]) + + +class TestCompileKeymap(unittest.TestCase): + def test_empty_keymap(self): + keymap = {} + result = compile_keymap(keymap) + self.assertEqual(result, {}) + + def test_single_keymap(self): + keymap = {b"a": "action"} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": "action"}) + + def test_nested_keymap(self): + keymap = {b"a": {b"b": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action"}}) + + def test_empty_value(self): + keymap = {b"a": {b"": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action"}}) + + def test_multiple_empty_values(self): + keymap = {b"a": {b"": "action1", b"b": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action1", b"b": "action2"}}) + + def test_multiple_keymaps(self): + keymap = {b"a": {b"b": "action1", b"c": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action1", b"c": "action2"}}) + + def test_nested_multiple_keymaps(self): + keymap = {b"a": {b"b": {b"c": "action"}}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": {b"c": "action"}}}) + + +if __name__ == "__main__": unittest.main() From 7e57640c7ec6b7b5ce9b5eac465f6b771fd6ae69 Mon Sep 17 00:00:00 2001 From: Shauna Date: Mon, 20 May 2024 16:10:53 -0400 Subject: [PATCH 128/903] gh-118912: Remove description of issue fixed in 3.5 from autospeccing guide (#119232) * Remove description of issue fixed in 3.5 from autospeccing guide * Make autospeccing note text more succint and lint whitespace * Add linting changes (missed in last commit) --------- Co-authored-by: Carol Willing --- Doc/library/unittest.mock.rst | 40 +++++++---------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 0c1cecf45381e2..d8ba24c3146cf2 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2584,40 +2584,16 @@ called incorrectly. Before I explain how auto-speccing works, here's why it is needed. -:class:`Mock` is a very powerful and flexible object, but it suffers from two flaws -when used to mock out objects from a system under test. One of these flaws is -specific to the :class:`Mock` api and the other is a more general problem with using -mock objects. - -First the problem specific to :class:`Mock`. :class:`Mock` has two assert methods that are -extremely handy: :meth:`~Mock.assert_called_with` and -:meth:`~Mock.assert_called_once_with`. - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - Traceback (most recent call last): - ... - AssertionError: Expected 'mock' to be called once. Called 2 times. - -Because mocks auto-create attributes on demand, and allow you to call them -with arbitrary arguments, if you misspell one of these assert methods then -your assertion is gone: - -.. code-block:: pycon - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assret_called_once_with(4, 5, 6) # Intentional typo! +:class:`Mock` is a very powerful and flexible object, but it suffers from a flaw which +is general to mocking. If you refactor some of your code, rename members and so on, any +tests for code that is still using the *old api* but uses mocks instead of the real +objects will still pass. This means your tests can all pass even though your code is +broken. -Your tests can pass silently and incorrectly because of the typo. +.. versionchanged:: 3.5 -The second issue is more general to mocking. If you refactor some of your -code, rename members and so on, any tests for code that is still using the -*old api* but uses mocks instead of the real objects will still pass. This -means your tests can all pass even though your code is broken. + Before 3.5, tests with a typo in the word assert would silently pass when they should + raise an error. You can still achieve this behavior by passing ``unsafe=True`` to Mock. Note that this is another reason why you need integration tests as well as unit tests. Testing everything in isolation is all fine and dandy, but if you From 8231a24454c854ea22590fd74733d29e4274122d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mark=20Jason=20Dominus=20=28=E9=99=B6=E6=95=8F=E4=BF=AE=29?= Date: Mon, 20 May 2024 16:29:17 -0400 Subject: [PATCH 129/903] gh-94808: Add test coverage for "starred kind" in _PyPegen_set_expr_context (GH-119222) Add test coverage for "starred kind" in _PyPegen_set_expr_context --- Lib/test/test_unpack_ex.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index c201d08f61b8cd..9e2d54bd3a8c4e 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -26,6 +26,12 @@ >>> a == [7, 8, 9] True +Unpack nested implied tuple + + >>> [*[*a]] = [[7,8,9]] + >>> a == [[7,8,9]] + True + Unpack string... fun! >>> a, *b = 'one' From bf17986096491b9ca14c214ed4885340e7857e12 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 20 May 2024 13:39:30 -0700 Subject: [PATCH 130/903] gh-119253: use ImportError in _ios_support (#119254) --- Lib/_ios_support.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py index db3fe23e45bca0..20467a7c2bcaeb 100644 --- a/Lib/_ios_support.py +++ b/Lib/_ios_support.py @@ -5,7 +5,7 @@ # ctypes is an optional module. If it's not present, we're limited in what # we can tell about the system, but we don't want to prevent the module # from working. - print("ctypes isn't available; iOS system calls will not be available") + print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr) objc = None else: # ctypes is available. Load the ObjC library, and wrap the objc_getClass, @@ -13,7 +13,7 @@ lib = util.find_library("objc") if lib is None: # Failed to load the objc library - raise RuntimeError("ObjC runtime library couldn't be loaded") + raise ImportError("ObjC runtime library couldn't be loaded") objc = cdll.LoadLibrary(lib) objc.objc_getClass.restype = c_void_p From 9257731f5d3e9d4f99e314b23a14506563e167d7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 20 May 2024 17:05:39 -0400 Subject: [PATCH 131/903] gh-119050: Add XML support to libregrtest refleak checker (#119148) regrtest test runner: Add XML support to the refleak checker (-R option). * run_unittest() now stores XML elements as string, rather than objects, in support.junit_xml_list. * runtest_refleak() now saves/restores XML strings before/after checking for reference leaks. Save XML into a temporary file. --- Lib/test/libregrtest/cmdline.py | 9 ----- Lib/test/libregrtest/refleak.py | 35 +++++++++++++++++-- Lib/test/libregrtest/single.py | 9 ++--- Lib/test/test_regrtest.py | 9 ----- ...-05-18-10-59-27.gh-issue-119050.g4qiH7.rst | 2 ++ 5 files changed, 39 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-05-18-10-59-27.gh-issue-119050.g4qiH7.rst diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 3e7428c4ad3797..d4dac77b250ad6 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -520,15 +520,6 @@ def _parse_args(args, **kwargs): "--huntrleaks without -jN option", file=sys.stderr) - if ns.huntrleaks and ns.xmlpath: - # The XML data is written into a file outside runtest_refleak(), so - # it looks like a leak but it's not. Simply disable XML output when - # hunting for reference leaks (gh-83434). - ns.xmlpath = None - print("WARNING: Disable --junit-xml because it's incompatible " - "with --huntrleaks", - file=sys.stderr) - if ns.forever: # --forever implies --failfast ns.failfast = True diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index bb57ffd75636cb..85a5cb72083264 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,3 +1,4 @@ +import os import sys import warnings from inspect import isabstract @@ -23,6 +24,30 @@ def _get_dump(cls): cls._abc_negative_cache, cls._abc_negative_cache_version) +def save_support_xml(filename): + if support.junit_xml_list is None: + return + + import pickle + with open(filename, 'xb') as fp: + pickle.dump(support.junit_xml_list, fp) + support.junit_xml_list = None + + +def restore_support_xml(filename): + try: + fp = open(filename, 'rb') + except FileNotFoundError: + return + + import pickle + with fp: + xml_list = pickle.load(fp) + os.unlink(filename) + + support.junit_xml_list = xml_list + + def runtest_refleak(test_name, test_func, hunt_refleak: HuntRefleak, quiet: bool): @@ -95,7 +120,8 @@ def get_pooled_int(value): numbers = numbers[:warmups] + ':' + numbers[warmups:] print(numbers, file=sys.stderr, flush=True) - results = None + xml_filename = 'refleak-xml.tmp' + result = None dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() @@ -103,10 +129,11 @@ def get_pooled_int(value): current = refleak_helper._hunting_for_refleaks refleak_helper._hunting_for_refleaks = True try: - results = test_func() + result = test_func() finally: refleak_helper._hunting_for_refleaks = current + save_support_xml(xml_filename) dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() @@ -145,6 +172,8 @@ def get_pooled_int(value): fd_before = fd_after interned_before = interned_after + restore_support_xml(xml_filename) + if not quiet: print(file=sys.stderr) @@ -189,7 +218,7 @@ def check_fd_deltas(deltas): failed = True else: print(' (this is fine)', file=sys.stderr, flush=True) - return (failed, results) + return (failed, result) def dash_R_cleanup(fs, ps, pic, zdc, abcs): diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index fc2f2716ad4ce0..adc8f1f455579f 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -57,7 +57,10 @@ def _run_suite(suite): result = runner.run(suite) if support.junit_xml_list is not None: - support.junit_xml_list.append(result.get_xml_element()) + import xml.etree.ElementTree as ET + xml_elem = result.get_xml_element() + xml_str = ET.tostring(xml_elem).decode('ascii') + support.junit_xml_list.append(xml_str) if not result.testsRun and not result.skipped and not result.errors: raise support.TestDidNotRun @@ -280,9 +283,7 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: xml_list = support.junit_xml_list if xml_list: - import xml.etree.ElementTree as ET - result.xml_data = [ET.tostring(x).decode('us-ascii') - for x in xml_list] + result.xml_data = xml_list finally: if use_timeout: faulthandler.cancel_dump_traceback_later() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 809abd7e92d65f..17eff617a56aa4 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -473,15 +473,6 @@ def test_verbose3_huntrleaks(self): self.assertEqual(regrtest.hunt_refleak.runs, 10) self.assertFalse(regrtest.output_on_failure) - def test_xml_huntrleaks(self): - args = ['-R', '3:12', '--junit-xml', 'output.xml'] - with support.captured_stderr(): - regrtest = self.create_regrtest(args) - self.assertIsNotNone(regrtest.hunt_refleak) - self.assertEqual(regrtest.hunt_refleak.warmups, 3) - self.assertEqual(regrtest.hunt_refleak.runs, 12) - self.assertIsNone(regrtest.junit_filename) - @dataclasses.dataclass(slots=True) class Rerun: diff --git a/Misc/NEWS.d/next/Tests/2024-05-18-10-59-27.gh-issue-119050.g4qiH7.rst b/Misc/NEWS.d/next/Tests/2024-05-18-10-59-27.gh-issue-119050.g4qiH7.rst new file mode 100644 index 00000000000000..cfc70c16b2b279 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-05-18-10-59-27.gh-issue-119050.g4qiH7.rst @@ -0,0 +1,2 @@ +regrtest test runner: Add XML support to the refleak checker (-R option). +Patch by Victor Stinner. From fe921931a35b461fb81821a474b510f4d67c520b Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 20 May 2024 17:57:32 -0400 Subject: [PATCH 132/903] gh-111201: Add tests for unix console class in pyrepl (#118653) --- Lib/test/test_pyrepl.py | 292 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 290 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 1daf3a8b498119..c61c090755904f 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -7,7 +7,7 @@ from code import InteractiveConsole from functools import partial from unittest import TestCase -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch, ANY from test.support import requires from test.support.import_helper import import_module @@ -22,6 +22,7 @@ from _pyrepl.console import Console, Event from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig from _pyrepl.simple_interact import _strip_final_indent +from _pyrepl.unix_console import UnixConsole from _pyrepl.unix_eventqueue import EventQueue from _pyrepl.input import KeymapTranslator from _pyrepl.keymap import parse_keys, compile_keymap @@ -105,7 +106,8 @@ def handle_all_events( handle_events_narrow_console = partial( - handle_all_events, prepare_console=partial(prepare_mock_console, width=10) + handle_all_events, + prepare_console=partial(prepare_mock_console, width=10), ) @@ -1198,5 +1200,291 @@ def test_nested_multiple_keymaps(self): self.assertEqual(result, {b"a": {b"b": {b"c": "action"}}}) +def unix_console(events, **kwargs): + console = UnixConsole() + console.get_event = MagicMock(side_effect=events) + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +handle_events_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console), +) +handle_events_narrow_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, width=5), +) +handle_events_short_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, height=1), +) +handle_events_unix_console_height_3 = partial( + handle_all_events, prepare_console=partial(unix_console, height=3) +) + + +TERM_CAPABILITIES = { + "bel": b"\x07", + "civis": b"\x1b[?25l", + "clear": b"\x1b[H\x1b[2J", + "cnorm": b"\x1b[?12l\x1b[?25h", + "cub": b"\x1b[%p1%dD", + "cub1": b"\x08", + "cud": b"\x1b[%p1%dB", + "cud1": b"\n", + "cuf": b"\x1b[%p1%dC", + "cuf1": b"\x1b[C", + "cup": b"\x1b[%i%p1%d;%p2%dH", + "cuu": b"\x1b[%p1%dA", + "cuu1": b"\x1b[A", + "dch1": b"\x1b[P", + "dch": b"\x1b[%p1%dP", + "el": b"\x1b[K", + "hpa": b"\x1b[%i%p1%dG", + "ich": b"\x1b[%p1%d@", + "ich1": None, + "ind": b"\n", + "pad": None, + "ri": b"\x1bM", + "rmkx": b"\x1b[?1l\x1b>", + "smkx": b"\x1b[?1h\x1b=", +} + + +@patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) +@patch( + "_pyrepl.curses.tparm", + lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), +) +@patch("_pyrepl.curses.setupterm", lambda a, b: None) +@patch( + "termios.tcgetattr", + lambda _: [ + 27394, + 3, + 19200, + 536872399, + 38400, + 38400, + [ + b"\x04", + b"\xff", + b"\xff", + b"\x7f", + b"\x17", + b"\x15", + b"\x12", + b"\x00", + b"\x03", + b"\x1c", + b"\x1a", + b"\x19", + b"\x11", + b"\x13", + b"\x16", + b"\x0f", + b"\x01", + b"\x00", + b"\x14", + b"\x00", + ], + ], +) +@patch("termios.tcsetattr", lambda a, b, c: None) +@patch("os.write") +class TestConsole(TestCase): + def test_simple_addition(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"4") + + def test_wrap(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, _ = handle_events_narrow_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"\\") + _os_write.assert_any_call(ANY, b"\n") + _os_write.assert_any_call(ANY, b"4") + + def test_cursor_left(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + + def test_cursor_left_right(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuf"] + b":1") + + def test_cursor_up(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + + def test_cursor_up_down(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cud"] + b":1") + + def test_cursor_back_write(self, _os_write): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, b"2") + + def test_multiline_function_move_up_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, _ = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + + def test_multiline_function_move_up_down_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, _ = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ind"] + b":") + + def test_resize_bigger_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_short_unix_console(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, _ = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ri"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b"def f():"), + ] + ) + + def test_resize_smaller_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_unix_console_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, _ = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ind"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b" foo"), + ] + ) + + if __name__ == "__main__": unittest.main() From 423bbcbbc43cacfb6a217c04f890a47d3cf7c3a9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 20 May 2024 18:34:57 -0400 Subject: [PATCH 133/903] gh-108267 Fix another dataclasses docs typo (#119277) --- Doc/library/dataclasses.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 7aa754c9ccc0a1..045bf6277289d8 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -616,7 +616,8 @@ methods will raise a :exc:`FrozenInstanceError` when invoked. There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and must use :meth:`!object.__setattr__`. -.. Make sure to not remove "object" from "object.__setattr__" in the above markup + +.. Make sure to not remove "object" from "object.__setattr__" in the above markup! .. _dataclasses-inheritance: From 172690227e771c2e8ab137815073e3a172c08dec Mon Sep 17 00:00:00 2001 From: Melanie Arbor Date: Mon, 20 May 2024 19:38:33 -0400 Subject: [PATCH 134/903] gh-102136: Add -m to options that work with -i (GH-119271) * GH-102136: Add -m to options that work with -i * Linting --- Doc/using/cmdline.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 2d95fa9474033f..dcca9060899a0f 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -290,9 +290,15 @@ Miscellaneous options .. option:: -i - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. The + Enter interactive mode after execution. + + Using the :option:`-i` option will enter interactive mode in any of the following circumstances\: + + * When a script is passed as first argument + * When the :option:`-c` option is used + * When the :option:`-m` option is used + + Interactive mode will start even when :data:`sys.stdin` does not appear to be a terminal. The :envvar:`PYTHONSTARTUP` file is not read. This can be useful to inspect global variables or a stack trace when a script From 538ed5e4818aa0d0aa759634e8bfa23e317434a1 Mon Sep 17 00:00:00 2001 From: Wulian233 <71213467+Wulian233@users.noreply.github.com> Date: Tue, 21 May 2024 11:32:00 +0800 Subject: [PATCH 135/903] gh-119174: Fix high DPI causes turtledemo(turtle-graphics examples) windows blurry (#119175) ------ Co-authored-by: Terry Jan Reedy --- Lib/idlelib/config.py | 3 ++- Lib/idlelib/pyshell.py | 10 ++-------- Lib/idlelib/util.py | 15 +++++++++++++++ Lib/turtledemo/__main__.py | 6 ++++-- ...2024-05-19-18-49-04.gh-issue-119174.5GTv7d.rst | 3 +++ 5 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-19-18-49-04.gh-issue-119174.5GTv7d.rst diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 7fc08ef9748182..6a5acac9be8888 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -158,8 +158,9 @@ def __init__(self, _utest=False): self.defaultCfg = {} self.userCfg = {} self.cfg = {} # TODO use to select userCfg vs defaultCfg + + # See https://bugs.python.org/issue4630#msg356516 for following. # self.blink_off_time = ['insertofftime'] - # See https://bugs.python.org/issue4630#msg356516. if not _utest: self.CreateConfigHandlers() diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 1524fccd5d20f8..d8b2652d5d7979 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -11,15 +11,9 @@ "Your Python may not be configured for Tk. **", file=sys.__stderr__) raise SystemExit(1) -# Valid arguments for the ...Awareness call below are defined in the following. -# https://msdn.microsoft.com/en-us/library/windows/desktop/dn280512(v=vs.85).aspx if sys.platform == 'win32': - try: - import ctypes - PROCESS_SYSTEM_DPI_AWARE = 1 # Int required. - ctypes.OleDLL('shcore').SetProcessDpiAwareness(PROCESS_SYSTEM_DPI_AWARE) - except (ImportError, AttributeError, OSError): - pass + from idlelib.util import fix_win_hidpi + fix_win_hidpi() from tkinter import messagebox diff --git a/Lib/idlelib/util.py b/Lib/idlelib/util.py index a7ae74b0579004..e05604ab4853f6 100644 --- a/Lib/idlelib/util.py +++ b/Lib/idlelib/util.py @@ -12,11 +12,26 @@ * std streams (pyshell, run), * warning stuff (pyshell, run). """ +import sys # .pyw is for Windows; .pyi is for typing stub files. # The extension order is needed for iomenu open/save dialogs. py_extensions = ('.py', '.pyw', '.pyi') + +# Fix for HiDPI screens on Windows. CALL BEFORE ANY TK OPERATIONS! +# URL for arguments for the ...Awareness call below. +# https://msdn.microsoft.com/en-us/library/windows/desktop/dn280512(v=vs.85).aspx +if sys.platform == 'win32': # pragma: no cover + def fix_win_hidpi(): # Called in pyshell and turtledemo. + try: + import ctypes + PROCESS_SYSTEM_DPI_AWARE = 1 # Int required. + ctypes.OleDLL('shcore').SetProcessDpiAwareness(PROCESS_SYSTEM_DPI_AWARE) + except (ImportError, AttributeError, OSError): + pass + + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_util', verbosity=2) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index 2ab6c15e2c079e..731f98b02b17a7 100755 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -92,13 +92,15 @@ from idlelib.colorizer import ColorDelegator, color_config from idlelib.percolator import Percolator from idlelib.textview import view_text +import turtle from turtledemo import __doc__ as about_turtledemo -import turtle +if sys.platform == 'win32': + from idlelib.util import fix_win_hidpi + fix_win_hidpi() demo_dir = os.path.dirname(os.path.abspath(__file__)) darwin = sys.platform == 'darwin' - STARTUP = 1 READY = 2 RUNNING = 3 diff --git a/Misc/NEWS.d/next/Library/2024-05-19-18-49-04.gh-issue-119174.5GTv7d.rst b/Misc/NEWS.d/next/Library/2024-05-19-18-49-04.gh-issue-119174.5GTv7d.rst new file mode 100644 index 00000000000000..7b467b9ebd0d80 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-19-18-49-04.gh-issue-119174.5GTv7d.rst @@ -0,0 +1,3 @@ +Fix high DPI causes turtledemo(turtle-graphics examples) windows blurry +Patch by Wulian233 and Terry Jan Reedy + From e870c852c0ea96fa4e4569e9c39c7ceb80ce858d Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Tue, 21 May 2024 13:32:15 +1000 Subject: [PATCH 136/903] gh-74929: PEP 667 general docs update (gh-119201) * expand on What's New entry for PEP 667 (including porting notes) * define 'optimized scope' as a glossary term * cover comprehensions and generator expressions in locals() docs * review all mentions of "locals" in documentation (updating if needed) * review all mentions of "f_locals" in documentation (updating if needed) --- Doc/c-api/frame.rst | 11 ++-- Doc/glossary.rst | 9 ++++ Doc/library/code.rst | 6 +-- Doc/library/functions.rst | 103 ++++++++++++++++++++++++------------ Doc/library/pdb.rst | 22 ++++---- Doc/library/profile.rst | 2 +- Doc/library/traceback.rst | 2 +- Doc/reference/datamodel.rst | 2 +- Doc/whatsnew/3.13.rst | 55 +++++++++++++++++-- Lib/code.py | 8 +-- Lib/pdb.py | 9 ++-- 11 files changed, 163 insertions(+), 66 deletions(-) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 82e0980ad753d0..638a740e0c24da 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -121,17 +121,18 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame) Get the *frame*'s :attr:`~frame.f_locals` attribute. - If the frame refers to a function or comprehension, this returns - a write-through proxy object that allows modifying the locals. - In all other cases (classes, modules) it returns the :class:`dict` - representing the frame locals directly. + If the frame refers to an :term:`optimized scope`, this returns a + write-through proxy object that allows modifying the locals. + In all other cases (classes, modules, :func:`exec`, :func:`eval`) it returns + the mapping representing the frame locals directly (as described for + :func:`locals`). Return a :term:`strong reference`. .. versionadded:: 3.11 .. versionchanged:: 3.13 - Return a proxy object for functions and comprehensions. + As part of :pep:`667`, return a proxy object for optimized scopes. .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 2846f77feb112d..42e2a6f4e301b6 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -889,6 +889,15 @@ Glossary (methods). Also the ultimate base class of any :term:`new-style class`. + optimized scope + A scope where target local variable names are reliably known to the + compiler when the code is compiled, allowing optimization of read and + write access to these names. The local namespaces for functions, + generators, coroutines, comprehensions, and generator expressions are + optimized in this fashion. Note: most interpreter optimizations are + applied to all scopes, only those relying on a known set of local + and nonlocal variable names are restricted to optimized scopes. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 8c3a3e8e95a11f..8f7692df9fb22d 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -18,9 +18,9 @@ build applications which provide an interactive interpreter prompt. This class deals with parsing and interpreter state (the user's namespace); it does not deal with input buffering or prompting or input file naming (the filename is always passed in explicitly). The optional *locals* argument - specifies the dictionary in which code will be executed; it defaults to a newly - created dictionary with key ``'__name__'`` set to ``'__console__'`` and key - ``'__doc__'`` set to ``None``. + specifies a mapping to use as the namespace in which code will be executed; + it defaults to a newly created dictionary with key ``'__name__'`` set to + ``'__console__'`` and key ``'__doc__'`` set to ``None``. .. class:: InteractiveConsole(locals=None, filename="", local_exit=False) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3986ace02b561a..a879ddbca92e82 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -543,18 +543,19 @@ are always available. They are listed here in alphabetical order. The *expression* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* - dictionaries as global and local namespace. If the *globals* dictionary is + mappings as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is inserted under that key before *expression* is parsed. That way you can control what builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* dictionary is omitted it defaults to the - *globals* dictionary. If both dictionaries are omitted, the expression is + :func:`eval`. If the *locals* mapping is omitted it defaults to the + *globals* dictionary. If both mappings are omitted, the expression is executed with the *globals* and *locals* in the environment where - :func:`eval` is called. Note, *eval()* does not have access to the + :func:`eval` is called. Note, *eval()* will only have access to the :term:`nested scopes ` (non-locals) in the enclosing - environment. + environment if they are already referenced in the scope that is calling + :func:`eval` (e.g. via a :keyword:`nonlocal` statement). Example: @@ -587,6 +588,11 @@ are always available. They are listed here in alphabetical order. The *globals* and *locals* arguments can now be passed as keywords. + .. versionchanged:: 3.13 + + The semantics of the default *locals* namespace have been adjusted as + described for the :func:`locals` builtin. + .. index:: pair: built-in function; exec .. function:: exec(source, /, globals=None, locals=None, *, closure=None) @@ -612,9 +618,15 @@ are always available. They are listed here in alphabetical order. .. note:: - Most users should just pass a *globals* argument and never *locals*. - If exec gets two separate objects as *globals* and *locals*, the code - will be executed as if it were embedded in a class definition. + When ``exec`` gets two separate objects as *globals* and *locals*, the + code will be executed as if it were embedded in a class definition. This + means functions and classes defined in the executed code will not be able + to access variables assigned at the top level (as the "top level" + variables are treated as class variables in a class definition). + Passing a :class:`collections.ChainMap` instance as *globals* allows name + lookups to be chained across multiple mappings without triggering this + behaviour. Values assigned to top level names in the executed code can be + retrieved by passing an empty dictionary as the first entry in the chain. If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module @@ -635,7 +647,7 @@ are always available. They are listed here in alphabetical order. .. note:: The built-in functions :func:`globals` and :func:`locals` return the current - global and local dictionary, respectively, which may be useful to pass around + global and local namespace, respectively, which may be useful to pass around for use as the second and third argument to :func:`exec`. .. note:: @@ -651,6 +663,11 @@ are always available. They are listed here in alphabetical order. The *globals* and *locals* arguments can now be passed as keywords. + .. versionchanged:: 3.13 + + The semantics of the default *locals* namespace have been adjusted as + described for the :func:`locals` builtin. + .. function:: filter(function, iterable) @@ -1056,39 +1073,51 @@ are always available. They are listed here in alphabetical order. variable names as the keys, and their currently bound references as the values. - At module scope, as well as when using ``exec()`` or ``eval()`` with a - single namespace, this function returns the same namespace as ``globals()``. + At module scope, as well as when using :func:`exec` or :func:`eval` with + a single namespace, this function returns the same namespace as + :func:`globals`. At class scope, it returns the namespace that will be passed to the metaclass constructor. When using ``exec()`` or ``eval()`` with separate local and global - namespaces, it returns the local namespace passed in to the function call. + arguments, it returns the local namespace passed in to the function call. In all of the above cases, each call to ``locals()`` in a given frame of execution will return the *same* mapping object. Changes made through - the mapping object returned from ``locals()`` will be visible as bound, - rebound, or deleted local variables, and binding, rebinding, or deleting - local variables will immediately affect the contents of the returned mapping - object. - - At function scope (including for generators and coroutines), each call to - ``locals()`` instead returns a fresh dictionary containing the current - bindings of the function's local variables and any nonlocal cell references. - In this case, name binding changes made via the returned dict are *not* - written back to the corresponding local variables or nonlocal cell - references, and binding, rebinding, or deleting local variables and nonlocal - cell references does *not* affect the contents of previously returned - dictionaries. + the mapping object returned from ``locals()`` will be visible as assigned, + reassigned, or deleted local variables, and assigning, reassigning, or + deleting local variables will immediately affect the contents of the + returned mapping object. + + In an :term:`optimized scope` (including functions, generators, and + coroutines), each call to ``locals()`` instead returns a fresh dictionary + containing the current bindings of the function's local variables and any + nonlocal cell references. In this case, name binding changes made via the + returned dict are *not* written back to the corresponding local variables + or nonlocal cell references, and assigning, reassigning, or deleting local + variables and nonlocal cell references does *not* affect the contents + of previously returned dictionaries. + + Calling ``locals()`` as part of a comprehension in a function, generator, or + coroutine is equivalent to calling it in the containing scope, except that + the comprehension's initialised iteration variables will be included. In + other scopes, it behaves as if the comprehension were running as a nested + function. + + Calling ``locals()`` as part of a generator expression is equivalent to + calling it in a nested generator function. + + .. versionchanged:: 3.12 + The behaviour of ``locals()`` in a comprehension has been updated as + described in :pep:`709`. .. versionchanged:: 3.13 - In previous versions, the semantics of mutating the mapping object - returned from this function were formally undefined. In CPython - specifically, the mapping returned at function scope could be - implicitly refreshed by other operations, such as calling ``locals()`` - again. Obtaining the legacy CPython behaviour now requires explicit - calls to update the initially returned dictionary with the results - of subsequent calls to ``locals()``. + As part of :pep:`667`, the semantics of mutating the mapping objects + returned from this function are now defined. The behavior in + :term:`optimized scopes ` is now as described above. + Aside from being defined, the behaviour in other scopes remains + unchanged from previous versions. .. function:: map(function, iterable, *iterables) @@ -1975,14 +2004,18 @@ are always available. They are listed here in alphabetical order. :attr:`~object.__dict__` attributes (for example, classes use a :class:`types.MappingProxyType` to prevent direct dictionary updates). - Without an argument, :func:`vars` acts like :func:`locals`. Note, the - locals dictionary is only useful for reads since updates to the locals - dictionary are ignored. + Without an argument, :func:`vars` acts like :func:`locals`. A :exc:`TypeError` exception is raised if an object is specified but it doesn't have a :attr:`~object.__dict__` attribute (for example, if its class defines the :attr:`~object.__slots__` attribute). + .. versionchanged:: 3.13 + + The result of calling this function without an argument has been + updated as described for the :func:`locals` builtin. + + .. function:: zip(*iterables, strict=False) Iterate over several iterables in parallel, producing tuples with an item diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 7a47a7d5d6754f..7d67e06434b799 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -123,6 +123,11 @@ The typical usage to inspect a crashed program is:: 0 (Pdb) +.. versionchanged:: 3.13 + The implementation of :pep:`667` means that name assignments made via ``pdb`` + will immediately affect the active scope, even when running inside an + :term:`optimized scope`. + The module defines the following functions; each enters the debugger in a slightly different way: @@ -579,18 +584,17 @@ can be overridden by the local file. .. pdbcommand:: interact - Start an interactive interpreter (using the :mod:`code` module) whose global - namespace contains all the (global and local) names found in the current - scope. Use ``exit()`` or ``quit()`` to exit the interpreter and return to - the debugger. + Start an interactive interpreter (using the :mod:`code` module) in a new + global namespace initialised from the local and global namespaces for the + current scope. Use ``exit()`` or ``quit()`` to exit the interpreter and + return to the debugger. .. note:: - Because interact creates a new global namespace with the current global - and local namespace for execution, assignment to variables will not - affect the original namespaces. - However, modification to the mutable objects will be reflected in the - original namespaces. + As ``interact`` creates a new dedicated namespace for code execution, + assignments to variables will not affect the original namespaces. + However, modifications to any referenced mutable objects will be reflected + in the original namespaces as usual. .. versionadded:: 3.2 diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 3ca802e024bc27..9721da7220d54d 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -234,7 +234,7 @@ functions: .. function:: runctx(command, globals, locals, filename=None, sort=-1) This function is similar to :func:`run`, with added arguments to supply the - globals and locals dictionaries for the *command* string. This routine + globals and locals mappings for the *command* string. This routine executes:: exec(command, globals, locals) diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 9983b8da427a2a..bfd2c3efc4b1f6 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -473,7 +473,7 @@ in a :ref:`traceback `. attribute accessed (which also happens when casting it to a :class:`tuple`). :attr:`~FrameSummary.line` may be directly provided, and will prevent line lookups happening at all. *locals* is an optional local variable - dictionary, and if supplied the variable representations are stored in the + mapping, and if supplied the variable representations are stored in the summary for later display. :class:`!FrameSummary` instances have the following attributes: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index d3e066797f8837..0fe9681f93f135 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1349,7 +1349,7 @@ Special read-only attributes * - .. attribute:: frame.f_locals - The dictionary used by the frame to look up :ref:`local variables `. - If the frame refers to a function or comprehension, + If the frame refers to an :term:`optimized scope`, this may return a write-through proxy object. .. versionchanged:: 3.13 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index dd5190d63ec579..8cd174f01ff600 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -94,10 +94,11 @@ Interpreter improvements: Performance improvements are modest -- we expect to be improving this over the next few releases. -* :pep:`667`: :attr:`FrameType.f_locals ` when used in - a function now returns a write-through proxy to the frame's locals, - rather than a ``dict``. See the PEP for corresponding C API changes - and deprecations. +* :pep:`667`: The :func:`locals` builtin now has + :ref:`defined semantics ` when mutating the + returned mapping. Python debuggers and similar tools may now more reliably + update local variables in optimized frames even during concurrent code + execution. New typing features: @@ -247,6 +248,34 @@ Improved Error Messages through ``self.X`` from any function in its body. (Contributed by Irit Katriel in :gh:`115775`.) +.. _whatsnew313-locals-semantics: + +Defined mutation semantics for ``locals()`` +------------------------------------------- + +Historically, the expected result of mutating the return value of :func:`locals` +has been left to individual Python implementations to define. + +Through :pep:`667`, Python 3.13 standardises the historical behaviour of CPython +for most code execution scopes, but changes +:term:`optimized scopes ` (functions, generators, coroutines, +comprehensions, and generator expressions) to explicitly return independent +snapshots of the currently assigned local variables, including locally +referenced nonlocal variables captured in closures. + +To ensure debuggers and similar tools can reliably update local variables in +scopes affected by this change, :attr:`FrameType.f_locals ` now +returns a write-through proxy to the frame's local and locally referenced +nonlocal variables in these scopes, rather than returning an inconsistently +updated shared ``dict`` instance with undefined runtime semantics. + +See :pep:`667` for more details, including related C API changes and +deprecations. + +(PEP and implementation contributed by Mark Shannon and Tian Gao in +:gh:`74929`. Documentation updates provided by Guido van Rossum and +Alyssa Coghlan.) + Incremental Garbage Collection ------------------------------ @@ -2177,6 +2206,24 @@ Changes in the Python API returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. (Contributed by Serhiy Storchaka in :gh:`115961`.) +* Calling :func:`locals` in an :term:`optimized scope` now produces an + independent snapshot on each call, and hence no longer implicitly updates + previously returned references. Obtaining the legacy CPython behaviour now + requires explicit calls to update the initially returned dictionary with the + results of subsequent calls to ``locals()``. (Changed as part of :pep:`667`.) + +* Calling :func:`locals` from a comprehension at module or class scope + (including via ``exec`` or ``eval``) once more behaves as if the comprehension + were running as an independent nested function (i.e. the local variables from + the containing scope are not included). In Python 3.12, this had changed + to include the local variables from the containing scope when implementing + :pep:`709`. (Changed as part of :pep:`667`.) + +* Accessing :attr:`FrameType.f_locals ` in an + :term:`optimized scope` now returns a write-through proxy rather than a + snapshot that gets updated at ill-specified times. If a snapshot is desired, + it must be created explicitly with ``dict`` or the proxy's ``.copy()`` method. + (Changed as part of :pep:`667`.) Changes in the C API -------------------- diff --git a/Lib/code.py b/Lib/code.py index 9d124563f728c2..0c2fd2963b2118 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -25,10 +25,10 @@ class InteractiveInterpreter: def __init__(self, locals=None): """Constructor. - The optional 'locals' argument specifies the dictionary in - which code will be executed; it defaults to a newly created - dictionary with key "__name__" set to "__console__" and key - "__doc__" set to None. + The optional 'locals' argument specifies a mapping to use as the + namespace in which code will be executed; it defaults to a newly + created dictionary with key "__name__" set to "__console__" and + key "__doc__" set to None. """ if locals is None: diff --git a/Lib/pdb.py b/Lib/pdb.py index e507a9bb896611..0f791d71950099 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -392,9 +392,12 @@ def setup(self, f, tb): self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] - # The f_locals dictionary is updated from the actual frame - # locals whenever the .f_locals accessor is called, so we - # cache it here to ensure that modifications are not overwritten. + # The f_locals dictionary used to be updated from the actual frame + # locals whenever the .f_locals accessor was called, so it was + # cached here to ensure that modifications were not overwritten. While + # the caching is no longer required now that f_locals is a direct proxy + # on optimized frames, it's also harmless, so the code structure has + # been left unchanged. self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) From b36533290608aed757f6eb16869a679650d32e17 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 21 May 2024 14:52:43 +0300 Subject: [PATCH 137/903] Use `fail-fast: false` in `mypy.yml` (#119297) See docs about this setting: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast --- .github/workflows/mypy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 35996f237814ba..1b2d998182e0f7 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -34,6 +34,7 @@ concurrency: jobs: mypy: strategy: + fail-fast: false matrix: target: [ "Lib/_pyrepl", From 73f4a58d36b65ec650e8f00b2affc4a4d3195f0c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 21 May 2024 08:53:20 -0400 Subject: [PATCH 138/903] gh-119102: Fix REPL for dumb terminal (#119269) Use CAN_USE_PYREPL of _pyrepl.__main__ in the site module to decide if _pyrepl.write_history_file() can be used. --- Lib/site.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index b63447d6673f68..4ba078388a37b8 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -523,8 +523,9 @@ def register_readline(): pass def write_history(): + from _pyrepl.__main__ import CAN_USE_PYREPL try: - if os.getenv("PYTHON_BASIC_REPL"): + if os.getenv("PYTHON_BASIC_REPL") or not CAN_USE_PYREPL: readline.write_history_file(history) else: _pyrepl.readline.write_history_file(history) From ab4263a82abe8b684d8ad1edf7c7c6ec286ff756 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 21 May 2024 09:49:18 -0400 Subject: [PATCH 139/903] gh-119053: Implement the fast path for list.__getitem__ (gh-119112) --- Objects/listobject.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index 7070165014f137..d09bb6391034d1 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -351,7 +351,11 @@ list_item_impl(PyListObject *self, Py_ssize_t idx) if (!valid_index(idx, size)) { goto exit; } +#ifdef Py_GIL_DISABLED + item = _Py_NewRefWithLock(self->ob_item[idx]); +#else item = Py_NewRef(self->ob_item[idx]); +#endif exit: Py_END_CRITICAL_SECTION(); return item; @@ -656,14 +660,15 @@ list_item(PyObject *aa, Py_ssize_t i) return NULL; } PyObject *item; - Py_BEGIN_CRITICAL_SECTION(a); #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread((PyObject *)a) && !_PyObject_GC_IS_SHARED(a)) { - _PyObject_GC_SET_SHARED(a); + item = list_get_item_ref(a, i); + if (item == NULL) { + PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); + return NULL; } -#endif +#else item = Py_NewRef(a->ob_item[i]); - Py_END_CRITICAL_SECTION(); +#endif return item; } From c4722cd0573c83aaa52b63a27022b9048a949f54 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 21 May 2024 08:00:52 -0700 Subject: [PATCH 140/903] GH-119292: Add job to JIT CI to build and test with --disable-gil (GH-119293) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/jit.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 2ec04da16946ff..b7938a177c856f 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -150,3 +150,21 @@ jobs: ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + jit-with-disabled-gil: + name: Free-Threaded (Debug) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT enabled and GIL disabled + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 18 + export PATH="$(llvm-config-18 --bindir):$PATH" + ./configure --enable-experimental-jit --with-pydebug --disable-gil + make all --jobs 4 + - name: Run tests + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From 0398d9339217aa0710c0de45a7e9b587136e7129 Mon Sep 17 00:00:00 2001 From: Alastair Stanley Date: Tue, 21 May 2024 12:17:41 -0400 Subject: [PATCH 141/903] =?UTF-8?q?gh-119035:=20Add=20Ctrl+=E2=86=90=20and?= =?UTF-8?q?=20Ctrl+=E2=86=92=20word-skipping=20keybindings=20to=20new=20re?= =?UTF-8?q?pl=20(#119248)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add word-skipping ctrl keybindings to new repl --- Lib/_pyrepl/keymap.py | 9 ++++++--- Lib/_pyrepl/reader.py | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py index e1421730e75717..a303589280e7f1 100644 --- a/Lib/_pyrepl/keymap.py +++ b/Lib/_pyrepl/keymap.py @@ -177,9 +177,12 @@ def _parse_key1(key, s): ret = key[s] s += 1 if ctrl: - if len(ret) > 1: - raise KeySpecError("\\C- must be followed by a character") - ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl() + if len(ret) == 1: + ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl() + elif ret in {"left", "right"}: + ret = f"ctrl {ret}" + else: + raise KeySpecError("\\C- followed by invalid key") if meta: ret = ["\033", ret] else: diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index d15a150180811d..2c8c9e7dc4b5da 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -136,7 +136,9 @@ def make_default_commands() -> dict[CommandName, type[Command]]: (r"\", "up"), (r"\", "down"), (r"\", "left"), + (r"\C-\", "backward-word"), (r"\", "right"), + (r"\C-\", "forward-word"), (r"\", "delete"), (r"\", "backspace"), (r"\M-\", "backward-kill-word"), From 62a29be5bb01c2d0f72d8f9b1b5539816e65310c Mon Sep 17 00:00:00 2001 From: Daniel Williams Date: Tue, 21 May 2024 12:23:50 -0400 Subject: [PATCH 142/903] gh-110383: Document `socket.makefile()` accepts combined modes (#119150) The supported mode values are 'r', 'w', and 'b', or a combination of those. --- Doc/library/socket.rst | 3 ++- Lib/socket.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 6405f7f00fcb21..2df0257d1f24f0 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1589,7 +1589,8 @@ to sockets. Return a :term:`file object` associated with the socket. The exact returned type depends on the arguments given to :meth:`makefile`. These arguments are interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'`` and ``'b'``. + the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or + a combination of those. The socket must be in blocking mode; it can have a timeout, but the file object's internal buffer may end up in an inconsistent state if a timeout diff --git a/Lib/socket.py b/Lib/socket.py index 77986fc2e48099..524ce1361b9091 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -306,7 +306,8 @@ def makefile(self, mode="r", buffering=None, *, """makefile(...) -> an I/O stream connected to the socket The arguments are as for io.open() after the filename, except the only - supported mode values are 'r' (default), 'w' and 'b'. + supported mode values are 'r' (default), 'w', 'b', or a combination of + those. """ # XXX refactor to share code? if not set(mode) <= {"r", "w", "b"}: From 9db2fd7edaa9d03e8c649c3bb0e8d963233cde22 Mon Sep 17 00:00:00 2001 From: Blaise Pabon Date: Tue, 21 May 2024 12:25:37 -0400 Subject: [PATCH 143/903] GH-110383: Improve Tutorial for Input Ouput (#119230) Co-authored-by: edson duarte --- Doc/tutorial/inputoutput.rst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index fe9ca9ccb9c7e0..857068a51ab843 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -37,16 +37,23 @@ printing space-separated values. There are several ways to format output. * The :meth:`str.format` method of strings requires more manual effort. You'll still use ``{`` and ``}`` to mark where a variable will be substituted and can provide detailed formatting directives, - but you'll also need to provide the information to be formatted. + but you'll also need to provide the information to be formatted. In the following code + block there are two examples of how to format variables: + :: >>> yes_votes = 42_572_654 - >>> no_votes = 43_132_495 - >>> percentage = yes_votes / (yes_votes + no_votes) + >>> total_votes = 85_705_149 + >>> percentage = yes_votes / total_votes >>> '{:-9} YES votes {:2.2%}'.format(yes_votes, percentage) ' 42572654 YES votes 49.67%' + Notice how the ``yes_votes`` are padded with spaces and a negative sign only for negative numbers. + The example also prints ``percentage`` multiplied by 100, with 2 decimal + places and followed by a percent sign (see :ref:`formatspec` for details). + + * Finally, you can do all the string handling yourself by using string slicing and concatenation operations to create any layout you can imagine. The string type has some methods that perform useful operations for padding @@ -197,7 +204,12 @@ notation. :: Jack: 4098; Sjoerd: 4127; Dcab: 8637678 This is particularly useful in combination with the built-in function -:func:`vars`, which returns a dictionary containing all local variables. +:func:`vars`, which returns a dictionary containing all local variables:: + + >>> table = {k: str(v) for k, v in vars().items()} + >>> message = " ".join([f'{k}: ' + '{' + k +'};' for k in table.keys()]) + >>> print(message.format(**table)) + __name__: __main__; __doc__: None; __package__: None; __loader__: ... As an example, the following lines produce a tidily aligned set of columns giving integers and their squares and cubes:: From e03dde5a24d3953e0b16f7cdefdc8b00aa9d9e11 Mon Sep 17 00:00:00 2001 From: Yan Yanchii Date: Tue, 21 May 2024 18:28:21 +0200 Subject: [PATCH 144/903] gh-113978: Ignore warnings on text completion inside REPL (#113979) --- Lib/rlcompleter.py | 10 ++++++---- .../2024-01-12-08-51-03.gh-issue-113978.MqTgB0.rst | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-01-12-08-51-03.gh-issue-113978.MqTgB0.rst diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 206d6fb511cdf6..23eb0020f42e8a 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -35,6 +35,7 @@ import keyword import re import __main__ +import warnings __all__ = ["Completer"] @@ -88,10 +89,11 @@ def complete(self, text, state): return None if state == 0: - if "." in text: - self.matches = self.attr_matches(text) - else: - self.matches = self.global_matches(text) + with warnings.catch_warnings(action="ignore"): + if "." in text: + self.matches = self.attr_matches(text) + else: + self.matches = self.global_matches(text) try: return self.matches[state] except IndexError: diff --git a/Misc/NEWS.d/next/Library/2024-01-12-08-51-03.gh-issue-113978.MqTgB0.rst b/Misc/NEWS.d/next/Library/2024-01-12-08-51-03.gh-issue-113978.MqTgB0.rst new file mode 100644 index 00000000000000..b8f9f255e0a75d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-12-08-51-03.gh-issue-113978.MqTgB0.rst @@ -0,0 +1 @@ +Ignore warnings on text completion inside REPL. From f49df4f486e531ff2666eb22854117c564b3de3d Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Tue, 21 May 2024 12:44:09 -0400 Subject: [PATCH 145/903] gh-119306: Break up _pyrepl tests (#119307) --- Lib/test/test_pyrepl.py | 1490 ------------------ Lib/test/test_pyrepl/__init__.py | 14 + Lib/test/test_pyrepl/__main__.py | 4 + Lib/test/test_pyrepl/support.py | 141 ++ Lib/test/test_pyrepl/test_input.py | 102 ++ Lib/test/test_pyrepl/test_keymap.py | 74 + Lib/test/test_pyrepl/test_pyrepl.py | 639 ++++++++ Lib/test/test_pyrepl/test_reader.py | 135 ++ Lib/test/test_pyrepl/test_unix_console.py | 294 ++++ Lib/test/test_pyrepl/test_unix_eventqueue.py | 113 ++ Makefile.pre.in | 1 + 11 files changed, 1517 insertions(+), 1490 deletions(-) delete mode 100644 Lib/test/test_pyrepl.py create mode 100644 Lib/test/test_pyrepl/__init__.py create mode 100644 Lib/test/test_pyrepl/__main__.py create mode 100644 Lib/test/test_pyrepl/support.py create mode 100644 Lib/test/test_pyrepl/test_input.py create mode 100644 Lib/test/test_pyrepl/test_keymap.py create mode 100644 Lib/test/test_pyrepl/test_pyrepl.py create mode 100644 Lib/test/test_pyrepl/test_reader.py create mode 100644 Lib/test/test_pyrepl/test_unix_console.py create mode 100644 Lib/test/test_pyrepl/test_unix_eventqueue.py diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py deleted file mode 100644 index c61c090755904f..00000000000000 --- a/Lib/test/test_pyrepl.py +++ /dev/null @@ -1,1490 +0,0 @@ -import itertools -import os -import rlcompleter -import sys -import tempfile -import unittest -from code import InteractiveConsole -from functools import partial -from unittest import TestCase -from unittest.mock import MagicMock, call, patch, ANY - -from test.support import requires -from test.support.import_helper import import_module - -# Optionally test pyrepl. This currently requires that the -# 'curses' resource be given on the regrtest command line using the -u -# option. Additionally, we need to attempt to import curses and readline. -requires("curses") -curses = import_module("curses") -readline = import_module("readline") - -from _pyrepl.console import Console, Event -from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig -from _pyrepl.simple_interact import _strip_final_indent -from _pyrepl.unix_console import UnixConsole -from _pyrepl.unix_eventqueue import EventQueue -from _pyrepl.input import KeymapTranslator -from _pyrepl.keymap import parse_keys, compile_keymap - - -def more_lines(unicodetext, namespace=None): - if namespace is None: - namespace = {} - src = _strip_final_indent(unicodetext) - console = InteractiveConsole(namespace, filename="") - try: - code = console.compile(src, "", "single") - except (OverflowError, SyntaxError, ValueError): - return False - else: - return code is None - - -def multiline_input(reader, namespace=None): - saved = reader.more_lines - try: - reader.more_lines = partial(more_lines, namespace=namespace) - reader.ps1 = reader.ps2 = ">>>" - reader.ps3 = reader.ps4 = "..." - return reader.readline() - finally: - reader.more_lines = saved - reader.paste_mode = False - - -def code_to_events(code): - for c in code: - yield Event(evt="key", data=c, raw=bytearray(c.encode("utf-8"))) - - -def prepare_mock_console(events, **kwargs): - console = MagicMock() - console.get_event.side_effect = events - console.height = 100 - console.width = 80 - for key, val in kwargs.items(): - setattr(console, key, val) - return console - - -def prepare_fake_console(**kwargs): - console = FakeConsole() - for key, val in kwargs.items(): - setattr(console, key, val) - return console - - -def prepare_reader(console, **kwargs): - config = ReadlineConfig(readline_completer=None) - reader = ReadlineAlikeReader(console=console, config=config) - reader.more_lines = partial(more_lines, namespace=None) - reader.paste_mode = True # Avoid extra indents - - def get_prompt(lineno, cursor_on_line) -> str: - return "" - - reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) - - for key, val in kwargs.items(): - setattr(reader, key, val) - - return reader - - -def handle_all_events( - events, prepare_console=prepare_mock_console, prepare_reader=prepare_reader -): - console = prepare_console(events) - reader = prepare_reader(console) - try: - while True: - reader.handle1() - except StopIteration: - pass - return reader, console - - -handle_events_narrow_console = partial( - handle_all_events, - prepare_console=partial(prepare_mock_console, width=10), -) - - -class FakeConsole(Console): - def __init__(self, events, encoding="utf-8"): - self.events = iter(events) - self.encoding = encoding - self.screen = [] - self.height = 100 - self.width = 80 - - def get_event(self, block: bool = True) -> Event | None: - return next(self.events) - - def getpending(self) -> Event: - return self.get_event(block=False) - - def getheightwidth(self) -> tuple[int, int]: - return self.height, self.width - - def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: - pass - - def prepare(self) -> None: - pass - - def restore(self) -> None: - pass - - def move_cursor(self, x: int, y: int) -> None: - pass - - def set_cursor_vis(self, visible: bool) -> None: - pass - - def push_char(self, char: int | bytes) -> None: - pass - - def beep(self) -> None: - pass - - def clear(self) -> None: - pass - - def finish(self) -> None: - pass - - def flushoutput(self) -> None: - pass - - def forgetinput(self) -> None: - pass - - def wait(self) -> None: - pass - - def repaint(self) -> None: - pass - - -class TestCursorPosition(TestCase): - def test_up_arrow_simple(self): - # fmt: off - code = ( - 'def f():\n' - ' ...\n' - ) - # fmt: on - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - ], - ) - - reader, console = handle_all_events(events) - self.assertEqual(reader.cxy, (0, 1)) - console.move_cursor.assert_called_once_with(0, 1) - - def test_down_arrow_end_of_input(self): - # fmt: off - code = ( - 'def f():\n' - ' ...\n' - ) - # fmt: on - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ], - ) - - reader, console = handle_all_events(events) - self.assertEqual(reader.cxy, (0, 2)) - console.move_cursor.assert_called_once_with(0, 2) - - def test_left_arrow_simple(self): - events = itertools.chain( - code_to_events("11+11"), - [ - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - ], - ) - - reader, console = handle_all_events(events) - self.assertEqual(reader.cxy, (4, 0)) - console.move_cursor.assert_called_once_with(4, 0) - - def test_right_arrow_end_of_line(self): - events = itertools.chain( - code_to_events("11+11"), - [ - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - ], - ) - - reader, console = handle_all_events(events) - self.assertEqual(reader.cxy, (5, 0)) - console.move_cursor.assert_called_once_with(5, 0) - - def test_cursor_position_simple_character(self): - events = itertools.chain(code_to_events("k")) - - reader, _ = handle_all_events(events) - self.assertEqual(reader.pos, 1) - - # 1 for simple character - self.assertEqual(reader.cxy, (1, 0)) - - def test_cursor_position_double_width_character(self): - events = itertools.chain(code_to_events("樂")) - - reader, _ = handle_all_events(events) - self.assertEqual(reader.pos, 1) - - # 2 for wide character - self.assertEqual(reader.cxy, (2, 0)) - - def test_cursor_position_double_width_character_move_left(self): - events = itertools.chain( - code_to_events("樂"), - [ - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - ], - ) - - reader, _ = handle_all_events(events) - self.assertEqual(reader.pos, 0) - self.assertEqual(reader.cxy, (0, 0)) - - def test_cursor_position_double_width_character_move_left_right(self): - events = itertools.chain( - code_to_events("樂"), - [ - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - ], - ) - - reader, _ = handle_all_events(events) - self.assertEqual(reader.pos, 1) - - # 2 for wide character - self.assertEqual(reader.cxy, (2, 0)) - - def test_cursor_position_double_width_characters_move_up(self): - for_loop = "for _ in _:" - - # fmt: off - code = ( - f"{for_loop}\n" - " ' 可口可乐; 可口可樂'" - ) - # fmt: on - - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - ], - ) - - reader, _ = handle_all_events(events) - - # cursor at end of first line - self.assertEqual(reader.pos, len(for_loop)) - self.assertEqual(reader.cxy, (len(for_loop), 0)) - - def test_cursor_position_double_width_characters_move_up_down(self): - for_loop = "for _ in _:" - - # fmt: off - code = ( - f"{for_loop}\n" - " ' 可口可乐; 可口可樂'" - ) - # fmt: on - - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ], - ) - - reader, _ = handle_all_events(events) - - # cursor here (showing 2nd line only): - # < ' 可口可乐; 可口可樂'> - # ^ - self.assertEqual(reader.pos, 19) - self.assertEqual(reader.cxy, (10, 1)) - - def test_cursor_position_multiple_double_width_characters_move_left(self): - events = itertools.chain( - code_to_events("' 可口可乐; 可口可樂'"), - [ - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - ], - ) - - reader, _ = handle_all_events(events) - self.assertEqual(reader.pos, 10) - - # 1 for quote, 1 for space, 2 per wide character, - # 1 for semicolon, 1 for space, 2 per wide character - self.assertEqual(reader.cxy, (16, 0)) - - def test_cursor_position_move_up_to_eol(self): - first_line = "for _ in _:" - second_line = " hello" - - # fmt: off - code = ( - f"{first_line}\n" - f"{second_line}\n" - " h\n" - " hel" - ) - # fmt: on - - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - ], - ) - - reader, _ = handle_all_events(events) - - # Cursor should be at end of line 1, even though line 2 is shorter - # for _ in _: - # hello - # h - # hel - self.assertEqual( - reader.pos, len(first_line) + len(second_line) + 1 - ) # +1 for newline - self.assertEqual(reader.cxy, (len(second_line), 1)) - - def test_cursor_position_move_down_to_eol(self): - last_line = " hel" - - # fmt: off - code = ( - "for _ in _:\n" - " hello\n" - " h\n" - f"{last_line}" - ) - # fmt: on - - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ], - ) - - reader, _ = handle_all_events(events) - - # Cursor should be at end of line 3, even though line 2 is shorter - # for _ in _: - # hello - # h - # hel - self.assertEqual(reader.pos, len(code)) - self.assertEqual(reader.cxy, (len(last_line), 3)) - - def test_cursor_position_multiple_mixed_lines_move_up(self): - # fmt: off - code = ( - "def foo():\n" - " x = '可口可乐; 可口可樂'\n" - " y = 'abckdfjskldfjslkdjf'" - ) - # fmt: on - - events = itertools.chain( - code_to_events(code), - 13 * [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], - [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], - ) - - reader, _ = handle_all_events(events) - - # By moving left, we're before the s: - # y = 'abckdfjskldfjslkdjf' - # ^ - # And we should move before the semi-colon despite the different offset - # x = '可口可乐; 可口可樂' - # ^ - self.assertEqual(reader.pos, 22) - self.assertEqual(reader.cxy, (15, 1)) - - def test_cursor_position_after_wrap_and_move_up(self): - # fmt: off - code = ( - "def foo():\n" - " hello" - ) - # fmt: on - - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - ], - ) - reader, _ = handle_events_narrow_console(events) - - # The code looks like this: - # def foo()\ - # : - # hello - # After moving up we should be after the colon in line 2 - self.assertEqual(reader.pos, 10) - self.assertEqual(reader.cxy, (1, 1)) - - -class TestPyReplOutput(TestCase): - def prepare_reader(self, events): - console = FakeConsole(events) - config = ReadlineConfig(readline_completer=None) - reader = ReadlineAlikeReader(console=console, config=config) - return reader - - def test_basic(self): - reader = self.prepare_reader(code_to_events("1+1\n")) - - output = multiline_input(reader) - self.assertEqual(output, "1+1") - - def test_multiline_edit(self): - events = itertools.chain( - code_to_events("def f():\n ...\n\n"), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), - Event(evt="key", data="g", raw=bytearray(b"g")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ], - ) - reader = self.prepare_reader(events) - - output = multiline_input(reader) - self.assertEqual(output, "def f():\n ...\n ") - output = multiline_input(reader) - self.assertEqual(output, "def g():\n ...\n ") - - def test_history_navigation_with_up_arrow(self): - events = itertools.chain( - code_to_events("1+1\n2+2\n"), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ], - ) - - reader = self.prepare_reader(events) - - output = multiline_input(reader) - self.assertEqual(output, "1+1") - output = multiline_input(reader) - self.assertEqual(output, "2+2") - output = multiline_input(reader) - self.assertEqual(output, "2+2") - output = multiline_input(reader) - self.assertEqual(output, "1+1") - - def test_history_navigation_with_down_arrow(self): - events = itertools.chain( - code_to_events("1+1\n2+2\n"), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ], - ) - - reader = self.prepare_reader(events) - - output = multiline_input(reader) - self.assertEqual(output, "1+1") - - def test_history_search(self): - events = itertools.chain( - code_to_events("1+1\n2+2\n3+3\n"), - [ - Event(evt="key", data="\x12", raw=bytearray(b"\x12")), - Event(evt="key", data="1", raw=bytearray(b"1")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - Event(evt="key", data="\n", raw=bytearray(b"\n")), - ], - ) - - reader = self.prepare_reader(events) - - output = multiline_input(reader) - self.assertEqual(output, "1+1") - output = multiline_input(reader) - self.assertEqual(output, "2+2") - output = multiline_input(reader) - self.assertEqual(output, "3+3") - output = multiline_input(reader) - self.assertEqual(output, "1+1") - - def test_control_character(self): - events = code_to_events("c\x1d\n") - reader = self.prepare_reader(events) - output = multiline_input(reader) - self.assertEqual(output, "c\x1d") - - -class TestPyReplCompleter(TestCase): - def prepare_reader(self, events, namespace): - console = FakeConsole(events) - config = ReadlineConfig() - config.readline_completer = rlcompleter.Completer(namespace).complete - reader = ReadlineAlikeReader(console=console, config=config) - return reader - - def test_simple_completion(self): - events = code_to_events("os.geten\t\n") - - namespace = {"os": os} - reader = self.prepare_reader(events, namespace) - - output = multiline_input(reader, namespace) - self.assertEqual(output, "os.getenv") - - def test_completion_with_many_options(self): - # Test with something that initially displays many options - # and then complete from one of them. The first time tab is - # pressed, the options are displayed (which corresponds to - # when the repl shows [ not unique ]) and the second completes - # from one of them. - events = code_to_events("os.\t\tO_AP\t\n") - - namespace = {"os": os} - reader = self.prepare_reader(events, namespace) - - output = multiline_input(reader, namespace) - self.assertEqual(output, "os.O_APPEND") - - def test_empty_namespace_completion(self): - events = code_to_events("os.geten\t\n") - namespace = {} - reader = self.prepare_reader(events, namespace) - - output = multiline_input(reader, namespace) - self.assertEqual(output, "os.geten") - - def test_global_namespace_completion(self): - events = code_to_events("py\t\n") - namespace = {"python": None} - reader = self.prepare_reader(events, namespace) - output = multiline_input(reader, namespace) - self.assertEqual(output, "python") - - def test_updown_arrow_with_completion_menu(self): - """Up arrow in the middle of unfinished tab completion when the menu is displayed - should work and trigger going back in history. Down arrow should subsequently - get us back to the incomplete command.""" - code = "import os\nos.\t\t" - namespace = {"os": os} - - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ], - code_to_events("\n"), - ) - reader = self.prepare_reader(events, namespace=namespace) - output = multiline_input(reader, namespace) - # This is the first line, nothing to see here - self.assertEqual(output, "import os") - # This is the second line. We pressed up and down arrows - # so we should end up where we were when we initiated tab completion. - output = multiline_input(reader, namespace) - self.assertEqual(output, "os.") - - -@patch("_pyrepl.curses.tigetstr", lambda x: b"") -class TestUnivEventQueue(TestCase): - def setUp(self): - self.file = tempfile.TemporaryFile() - - def tearDown(self) -> None: - self.file.close() - - def test_get(self): - eq = EventQueue(self.file.fileno(), "utf-8") - event = Event("key", "a", b"a") - eq.insert(event) - self.assertEqual(eq.get(), event) - - def test_empty(self): - eq = EventQueue(self.file.fileno(), "utf-8") - self.assertTrue(eq.empty()) - eq.insert(Event("key", "a", b"a")) - self.assertFalse(eq.empty()) - - def test_flush_buf(self): - eq = EventQueue(self.file.fileno(), "utf-8") - eq.buf.extend(b"test") - self.assertEqual(eq.flush_buf(), b"test") - self.assertEqual(eq.buf, bytearray()) - - def test_insert(self): - eq = EventQueue(self.file.fileno(), "utf-8") - event = Event("key", "a", b"a") - eq.insert(event) - self.assertEqual(eq.events[0], event) - - @patch("_pyrepl.unix_eventqueue.keymap") - def test_push_with_key_in_keymap(self, mock_keymap): - mock_keymap.compile_keymap.return_value = {"a": "b"} - eq = EventQueue(self.file.fileno(), "utf-8") - eq.keymap = {b"a": "b"} - eq.push("a") - mock_keymap.compile_keymap.assert_called() - self.assertEqual(eq.events[0].evt, "key") - self.assertEqual(eq.events[0].data, "b") - - @patch("_pyrepl.unix_eventqueue.keymap") - def test_push_without_key_in_keymap(self, mock_keymap): - mock_keymap.compile_keymap.return_value = {"a": "b"} - eq = EventQueue(self.file.fileno(), "utf-8") - eq.keymap = {b"c": "d"} - eq.push("a") - mock_keymap.compile_keymap.assert_called() - self.assertEqual(eq.events[0].evt, "key") - self.assertEqual(eq.events[0].data, "a") - - @patch("_pyrepl.unix_eventqueue.keymap") - def test_push_with_keymap_in_keymap(self, mock_keymap): - mock_keymap.compile_keymap.return_value = {"a": "b"} - eq = EventQueue(self.file.fileno(), "utf-8") - eq.keymap = {b"a": {b"b": "c"}} - eq.push("a") - mock_keymap.compile_keymap.assert_called() - self.assertTrue(eq.empty()) - eq.push("b") - self.assertEqual(eq.events[0].evt, "key") - self.assertEqual(eq.events[0].data, "c") - eq.push("d") - self.assertEqual(eq.events[1].evt, "key") - self.assertEqual(eq.events[1].data, "d") - - @patch("_pyrepl.unix_eventqueue.keymap") - def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): - mock_keymap.compile_keymap.return_value = {"a": "b"} - eq = EventQueue(self.file.fileno(), "utf-8") - eq.keymap = {b"a": {b"b": "c"}} - eq.push("a") - mock_keymap.compile_keymap.assert_called() - self.assertTrue(eq.empty()) - eq.flush_buf() - eq.push("\033") - self.assertEqual(eq.events[0].evt, "key") - self.assertEqual(eq.events[0].data, "\033") - eq.push("b") - self.assertEqual(eq.events[1].evt, "key") - self.assertEqual(eq.events[1].data, "b") - - def test_push_special_key(self): - eq = EventQueue(self.file.fileno(), "utf-8") - eq.keymap = {} - eq.push("\x1b") - eq.push("[") - eq.push("A") - self.assertEqual(eq.events[0].evt, "key") - self.assertEqual(eq.events[0].data, "\x1b") - - def test_push_unrecognized_escape_sequence(self): - eq = EventQueue(self.file.fileno(), "utf-8") - eq.keymap = {} - eq.push("\x1b") - eq.push("[") - eq.push("Z") - self.assertEqual(len(eq.events), 3) - self.assertEqual(eq.events[0].evt, "key") - self.assertEqual(eq.events[0].data, "\x1b") - self.assertEqual(eq.events[1].evt, "key") - self.assertEqual(eq.events[1].data, "[") - self.assertEqual(eq.events[2].evt, "key") - self.assertEqual(eq.events[2].data, "Z") - - -class TestPasteEvent(TestCase): - def prepare_reader(self, events): - console = FakeConsole(events) - config = ReadlineConfig(readline_completer=None) - reader = ReadlineAlikeReader(console=console, config=config) - return reader - - def test_paste(self): - # fmt: off - code = ( - 'def a():\n' - ' for x in range(10):\n' - ' if x%2:\n' - ' print(x)\n' - ' else:\n' - ' pass\n' - ) - # fmt: on - - events = itertools.chain( - [ - Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), - ], - code_to_events(code), - [ - Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), - ], - code_to_events("\n"), - ) - reader = self.prepare_reader(events) - output = multiline_input(reader) - self.assertEqual(output, code) - - def test_paste_mid_newlines(self): - # fmt: off - code = ( - 'def f():\n' - ' x = y\n' - ' \n' - ' y = z\n' - ) - # fmt: on - - events = itertools.chain( - [ - Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), - ], - code_to_events(code), - [ - Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), - ], - code_to_events("\n"), - ) - reader = self.prepare_reader(events) - output = multiline_input(reader) - self.assertEqual(output, code) - - def test_paste_mid_newlines_not_in_paste_mode(self): - # fmt: off - code = ( - 'def f():\n' - ' x = y\n' - ' \n' - ' y = z\n\n' - ) - - expected = ( - 'def f():\n' - ' x = y\n' - ' ' - ) - # fmt: on - - events = code_to_events(code) - reader = self.prepare_reader(events) - output = multiline_input(reader) - self.assertEqual(output, expected) - - def test_paste_not_in_paste_mode(self): - # fmt: off - input_code = ( - 'def a():\n' - ' for x in range(10):\n' - ' if x%2:\n' - ' print(x)\n' - ' else:\n' - ' pass\n\n' - ) - - output_code = ( - 'def a():\n' - ' for x in range(10):\n' - ' if x%2:\n' - ' print(x)\n' - ' else:' - ) - # fmt: on - - events = code_to_events(input_code) - reader = self.prepare_reader(events) - output = multiline_input(reader) - self.assertEqual(output, output_code) - - def test_bracketed_paste(self): - """Test that bracketed paste using \x1b[200~ and \x1b[201~ works.""" - # fmt: off - input_code = ( - 'def a():\n' - ' for x in range(10):\n' - '\n' - ' if x%2:\n' - ' print(x)\n' - '\n' - ' else:\n' - ' pass\n' - ) - - output_code = ( - 'def a():\n' - ' for x in range(10):\n' - '\n' - ' if x%2:\n' - ' print(x)\n' - '\n' - ' else:\n' - ' pass\n' - ) - # fmt: on - - paste_start = "\x1b[200~" - paste_end = "\x1b[201~" - - events = itertools.chain( - code_to_events(paste_start), - code_to_events(input_code), - code_to_events(paste_end), - code_to_events("\n"), - ) - reader = self.prepare_reader(events) - output = multiline_input(reader) - self.assertEqual(output, output_code) - - def test_bracketed_paste_single_line(self): - input_code = "oneline" - - paste_start = "\x1b[200~" - paste_end = "\x1b[201~" - - events = itertools.chain( - code_to_events(paste_start), - code_to_events(input_code), - code_to_events(paste_end), - code_to_events("\n"), - ) - reader = self.prepare_reader(events) - output = multiline_input(reader) - self.assertEqual(output, input_code) - - -class TestReader(TestCase): - def assert_screen_equals(self, reader, expected): - actual = reader.calc_screen() - expected = expected.split("\n") - self.assertListEqual(actual, expected) - - def test_calc_screen_wrap_simple(self): - events = code_to_events(10 * "a") - reader, _ = handle_events_narrow_console(events) - self.assert_screen_equals(reader, f"{9*"a"}\\\na") - - def test_calc_screen_wrap_wide_characters(self): - events = code_to_events(8 * "a" + "樂") - reader, _ = handle_events_narrow_console(events) - self.assert_screen_equals(reader, f"{8*"a"}\\\n樂") - - def test_calc_screen_wrap_three_lines(self): - events = code_to_events(20 * "a") - reader, _ = handle_events_narrow_console(events) - self.assert_screen_equals(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") - - def test_calc_screen_wrap_three_lines_mixed_character(self): - # fmt: off - code = ( - "def f():\n" - f" {8*"a"}\n" - f" {5*"樂"}" - ) - # fmt: on - - events = code_to_events(code) - reader, _ = handle_events_narrow_console(events) - - # fmt: off - self.assert_screen_equals(reader, ( - "def f():\n" - f" {7*"a"}\\\n" - "a\n" - f" {3*"樂"}\\\n" - "樂樂" - )) - # fmt: on - - def test_calc_screen_backspace(self): - events = itertools.chain( - code_to_events("aaa"), - [ - Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), - ], - ) - reader, _ = handle_all_events(events) - self.assert_screen_equals(reader, "aa") - - def test_calc_screen_wrap_removes_after_backspace(self): - events = itertools.chain( - code_to_events(10 * "a"), - [ - Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), - ], - ) - reader, _ = handle_events_narrow_console(events) - self.assert_screen_equals(reader, 9 * "a") - - def test_calc_screen_backspace_in_second_line_after_wrap(self): - events = itertools.chain( - code_to_events(11 * "a"), - [ - Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), - ], - ) - reader, _ = handle_events_narrow_console(events) - self.assert_screen_equals(reader, f"{9*"a"}\\\na") - - def test_setpos_for_xy_simple(self): - events = code_to_events("11+11") - reader, _ = handle_all_events(events) - reader.setpos_from_xy(0, 0) - self.assertEqual(reader.pos, 0) - - def test_setpos_from_xy_multiple_lines(self): - # fmt: off - code = ( - "def foo():\n" - " return 1" - ) - # fmt: on - - events = code_to_events(code) - reader, _ = handle_all_events(events) - reader.setpos_from_xy(2, 1) - self.assertEqual(reader.pos, 13) - - def test_setpos_from_xy_after_wrap(self): - # fmt: off - code = ( - "def foo():\n" - " hello" - ) - # fmt: on - - events = code_to_events(code) - reader, _ = handle_events_narrow_console(events) - reader.setpos_from_xy(2, 2) - self.assertEqual(reader.pos, 13) - - def test_setpos_fromxy_in_wrapped_line(self): - # fmt: off - code = ( - "def foo():\n" - " hello" - ) - # fmt: on - - events = code_to_events(code) - reader, _ = handle_events_narrow_console(events) - reader.setpos_from_xy(0, 1) - self.assertEqual(reader.pos, 9) - - def test_up_arrow_after_ctrl_r(self): - events = iter( - [ - Event(evt="key", data="\x12", raw=bytearray(b"\x12")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - ] - ) - - reader, _ = handle_all_events(events) - self.assert_screen_equals(reader, "") - - -class KeymapTranslatorTests(unittest.TestCase): - def test_push_single_key(self): - keymap = [("a", "command_a")] - translator = KeymapTranslator(keymap) - evt = Event("key", "a") - translator.push(evt) - result = translator.get() - self.assertEqual(result, ("command_a", ["a"])) - - def test_push_multiple_keys(self): - keymap = [("ab", "command_ab")] - translator = KeymapTranslator(keymap) - evt1 = Event("key", "a") - evt2 = Event("key", "b") - translator.push(evt1) - translator.push(evt2) - result = translator.get() - self.assertEqual(result, ("command_ab", ["a", "b"])) - - def test_push_invalid_key(self): - keymap = [("a", "command_a")] - translator = KeymapTranslator(keymap) - evt = Event("key", "b") - translator.push(evt) - result = translator.get() - self.assertEqual(result, (None, ["b"])) - - def test_push_invalid_key_with_stack(self): - keymap = [("ab", "command_ab")] - translator = KeymapTranslator(keymap) - evt1 = Event("key", "a") - evt2 = Event("key", "c") - translator.push(evt1) - translator.push(evt2) - result = translator.get() - self.assertEqual(result, (None, ["a", "c"])) - - def test_push_character_key(self): - keymap = [("a", "command_a")] - translator = KeymapTranslator(keymap) - evt = Event("key", "a") - translator.push(evt) - result = translator.get() - self.assertEqual(result, ("command_a", ["a"])) - - def test_push_character_key_with_stack(self): - keymap = [("ab", "command_ab")] - translator = KeymapTranslator(keymap) - evt1 = Event("key", "a") - evt2 = Event("key", "b") - evt3 = Event("key", "c") - translator.push(evt1) - translator.push(evt2) - translator.push(evt3) - result = translator.get() - self.assertEqual(result, ("command_ab", ["a", "b"])) - - def test_push_transition_key(self): - keymap = [("a", {"b": "command_ab"})] - translator = KeymapTranslator(keymap) - evt1 = Event("key", "a") - evt2 = Event("key", "b") - translator.push(evt1) - translator.push(evt2) - result = translator.get() - self.assertEqual(result, ("command_ab", ["a", "b"])) - - def test_push_transition_key_interrupted(self): - keymap = [("a", {"b": "command_ab"})] - translator = KeymapTranslator(keymap) - evt1 = Event("key", "a") - evt2 = Event("key", "c") - evt3 = Event("key", "b") - translator.push(evt1) - translator.push(evt2) - translator.push(evt3) - result = translator.get() - self.assertEqual(result, (None, ["a", "c"])) - - def test_push_invalid_key_with_unicode_category(self): - keymap = [("a", "command_a")] - translator = KeymapTranslator(keymap) - evt = Event("key", "\u0003") # Control character - translator.push(evt) - result = translator.get() - self.assertEqual(result, (None, ["\u0003"])) - - def test_empty(self): - keymap = [("a", "command_a")] - translator = KeymapTranslator(keymap) - self.assertTrue(translator.empty()) - evt = Event("key", "a") - translator.push(evt) - self.assertFalse(translator.empty()) - translator.get() - self.assertTrue(translator.empty()) - - -class TestParseKeys(unittest.TestCase): - def test_single_character(self): - self.assertEqual(parse_keys("a"), ["a"]) - self.assertEqual(parse_keys("b"), ["b"]) - self.assertEqual(parse_keys("1"), ["1"]) - - def test_escape_sequences(self): - self.assertEqual(parse_keys("\\n"), ["\n"]) - self.assertEqual(parse_keys("\\t"), ["\t"]) - self.assertEqual(parse_keys("\\\\"), ["\\"]) - self.assertEqual(parse_keys("\\'"), ["'"]) - self.assertEqual(parse_keys('\\"'), ['"']) - - def test_control_sequences(self): - self.assertEqual(parse_keys("\\C-a"), ["\x01"]) - self.assertEqual(parse_keys("\\C-b"), ["\x02"]) - self.assertEqual(parse_keys("\\C-c"), ["\x03"]) - - def test_meta_sequences(self): - self.assertEqual(parse_keys("\\M-a"), ["\033", "a"]) - self.assertEqual(parse_keys("\\M-b"), ["\033", "b"]) - self.assertEqual(parse_keys("\\M-c"), ["\033", "c"]) - - def test_keynames(self): - self.assertEqual(parse_keys("\\"), ["up"]) - self.assertEqual(parse_keys("\\"), ["down"]) - self.assertEqual(parse_keys("\\"), ["left"]) - self.assertEqual(parse_keys("\\"), ["right"]) - - def test_combinations(self): - self.assertEqual(parse_keys("\\C-a\\n\\"), ["\x01", "\n", "up"]) - self.assertEqual(parse_keys("\\M-a\\t\\"), ["\033", "a", "\t", "down"]) - - -class TestCompileKeymap(unittest.TestCase): - def test_empty_keymap(self): - keymap = {} - result = compile_keymap(keymap) - self.assertEqual(result, {}) - - def test_single_keymap(self): - keymap = {b"a": "action"} - result = compile_keymap(keymap) - self.assertEqual(result, {b"a": "action"}) - - def test_nested_keymap(self): - keymap = {b"a": {b"b": "action"}} - result = compile_keymap(keymap) - self.assertEqual(result, {b"a": {b"b": "action"}}) - - def test_empty_value(self): - keymap = {b"a": {b"": "action"}} - result = compile_keymap(keymap) - self.assertEqual(result, {b"a": {b"": "action"}}) - - def test_multiple_empty_values(self): - keymap = {b"a": {b"": "action1", b"b": "action2"}} - result = compile_keymap(keymap) - self.assertEqual(result, {b"a": {b"": "action1", b"b": "action2"}}) - - def test_multiple_keymaps(self): - keymap = {b"a": {b"b": "action1", b"c": "action2"}} - result = compile_keymap(keymap) - self.assertEqual(result, {b"a": {b"b": "action1", b"c": "action2"}}) - - def test_nested_multiple_keymaps(self): - keymap = {b"a": {b"b": {b"c": "action"}}} - result = compile_keymap(keymap) - self.assertEqual(result, {b"a": {b"b": {b"c": "action"}}}) - - -def unix_console(events, **kwargs): - console = UnixConsole() - console.get_event = MagicMock(side_effect=events) - - height = kwargs.get("height", 25) - width = kwargs.get("width", 80) - console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) - - console.prepare() - for key, val in kwargs.items(): - setattr(console, key, val) - return console - - -handle_events_unix_console = partial( - handle_all_events, - prepare_console=partial(unix_console), -) -handle_events_narrow_unix_console = partial( - handle_all_events, - prepare_console=partial(unix_console, width=5), -) -handle_events_short_unix_console = partial( - handle_all_events, - prepare_console=partial(unix_console, height=1), -) -handle_events_unix_console_height_3 = partial( - handle_all_events, prepare_console=partial(unix_console, height=3) -) - - -TERM_CAPABILITIES = { - "bel": b"\x07", - "civis": b"\x1b[?25l", - "clear": b"\x1b[H\x1b[2J", - "cnorm": b"\x1b[?12l\x1b[?25h", - "cub": b"\x1b[%p1%dD", - "cub1": b"\x08", - "cud": b"\x1b[%p1%dB", - "cud1": b"\n", - "cuf": b"\x1b[%p1%dC", - "cuf1": b"\x1b[C", - "cup": b"\x1b[%i%p1%d;%p2%dH", - "cuu": b"\x1b[%p1%dA", - "cuu1": b"\x1b[A", - "dch1": b"\x1b[P", - "dch": b"\x1b[%p1%dP", - "el": b"\x1b[K", - "hpa": b"\x1b[%i%p1%dG", - "ich": b"\x1b[%p1%d@", - "ich1": None, - "ind": b"\n", - "pad": None, - "ri": b"\x1bM", - "rmkx": b"\x1b[?1l\x1b>", - "smkx": b"\x1b[?1h\x1b=", -} - - -@patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) -@patch( - "_pyrepl.curses.tparm", - lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), -) -@patch("_pyrepl.curses.setupterm", lambda a, b: None) -@patch( - "termios.tcgetattr", - lambda _: [ - 27394, - 3, - 19200, - 536872399, - 38400, - 38400, - [ - b"\x04", - b"\xff", - b"\xff", - b"\x7f", - b"\x17", - b"\x15", - b"\x12", - b"\x00", - b"\x03", - b"\x1c", - b"\x1a", - b"\x19", - b"\x11", - b"\x13", - b"\x16", - b"\x0f", - b"\x01", - b"\x00", - b"\x14", - b"\x00", - ], - ], -) -@patch("termios.tcsetattr", lambda a, b, c: None) -@patch("os.write") -class TestConsole(TestCase): - def test_simple_addition(self, _os_write): - code = "12+34" - events = code_to_events(code) - _, _ = handle_events_unix_console(events) - _os_write.assert_any_call(ANY, b"1") - _os_write.assert_any_call(ANY, b"2") - _os_write.assert_any_call(ANY, b"+") - _os_write.assert_any_call(ANY, b"3") - _os_write.assert_any_call(ANY, b"4") - - def test_wrap(self, _os_write): - code = "12+34" - events = code_to_events(code) - _, _ = handle_events_narrow_unix_console(events) - _os_write.assert_any_call(ANY, b"1") - _os_write.assert_any_call(ANY, b"2") - _os_write.assert_any_call(ANY, b"+") - _os_write.assert_any_call(ANY, b"3") - _os_write.assert_any_call(ANY, b"\\") - _os_write.assert_any_call(ANY, b"\n") - _os_write.assert_any_call(ANY, b"4") - - def test_cursor_left(self, _os_write): - code = "1" - events = itertools.chain( - code_to_events(code), - [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], - ) - _, _ = handle_events_unix_console(events) - _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") - - def test_cursor_left_right(self, _os_write): - code = "1" - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - ], - ) - _, _ = handle_events_unix_console(events) - _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") - _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuf"] + b":1") - - def test_cursor_up(self, _os_write): - code = "1\n2+3" - events = itertools.chain( - code_to_events(code), - [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], - ) - _, _ = handle_events_unix_console(events) - _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") - - def test_cursor_up_down(self, _os_write): - code = "1\n2+3" - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - ], - ) - _, _ = handle_events_unix_console(events) - _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") - _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cud"] + b":1") - - def test_cursor_back_write(self, _os_write): - events = itertools.chain( - code_to_events("1"), - [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], - code_to_events("2"), - ) - _, _ = handle_events_unix_console(events) - _os_write.assert_any_call(ANY, b"1") - _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") - _os_write.assert_any_call(ANY, b"2") - - def test_multiline_function_move_up_short_terminal(self, _os_write): - # fmt: off - code = ( - "def f():\n" - " foo" - ) - # fmt: on - - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="scroll", data=None), - ], - ) - _, _ = handle_events_short_unix_console(events) - _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") - - def test_multiline_function_move_up_down_short_terminal(self, _os_write): - # fmt: off - code = ( - "def f():\n" - " foo" - ) - # fmt: on - - events = itertools.chain( - code_to_events(code), - [ - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="scroll", data=None), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="scroll", data=None), - ], - ) - _, _ = handle_events_short_unix_console(events) - _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") - _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ind"] + b":") - - def test_resize_bigger_on_multiline_function(self, _os_write): - # fmt: off - code = ( - "def f():\n" - " foo" - ) - # fmt: on - - events = itertools.chain(code_to_events(code)) - reader, console = handle_events_short_unix_console(events) - - console.height = 2 - console.getheightwidth = MagicMock(lambda _: (2, 80)) - - def same_reader(_): - return reader - - def same_console(events): - console.get_event = MagicMock(side_effect=events) - return console - - _, _ = handle_all_events( - [Event(evt="resize", data=None)], - prepare_reader=same_reader, - prepare_console=same_console, - ) - _os_write.assert_has_calls( - [ - call(ANY, TERM_CAPABILITIES["ri"] + b":"), - call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), - call(ANY, b"def f():"), - ] - ) - - def test_resize_smaller_on_multiline_function(self, _os_write): - # fmt: off - code = ( - "def f():\n" - " foo" - ) - # fmt: on - - events = itertools.chain(code_to_events(code)) - reader, console = handle_events_unix_console_height_3(events) - - console.height = 1 - console.getheightwidth = MagicMock(lambda _: (1, 80)) - - def same_reader(_): - return reader - - def same_console(events): - console.get_event = MagicMock(side_effect=events) - return console - - _, _ = handle_all_events( - [Event(evt="resize", data=None)], - prepare_reader=same_reader, - prepare_console=same_console, - ) - _os_write.assert_has_calls( - [ - call(ANY, TERM_CAPABILITIES["ind"] + b":"), - call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), - call(ANY, b" foo"), - ] - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py new file mode 100644 index 00000000000000..a9bc41f4d39f60 --- /dev/null +++ b/Lib/test/test_pyrepl/__init__.py @@ -0,0 +1,14 @@ +import os +from test.support import requires, load_package_tests +from test.support.import_helper import import_module + +# Optionally test pyrepl. This currently requires that the +# 'curses' resource be given on the regrtest command line using the -u +# option. Additionally, we need to attempt to import curses and readline. +requires("curses") +curses = import_module("curses") +readline = import_module("readline") + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pyrepl/__main__.py b/Lib/test/test_pyrepl/__main__.py new file mode 100644 index 00000000000000..cbe9e01d0df820 --- /dev/null +++ b/Lib/test/test_pyrepl/__main__.py @@ -0,0 +1,4 @@ +import unittest +from test.test_pyrepl import load_tests + +unittest.main() diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py new file mode 100644 index 00000000000000..75539049d43c2a --- /dev/null +++ b/Lib/test/test_pyrepl/support.py @@ -0,0 +1,141 @@ +from code import InteractiveConsole +from functools import partial +from typing import Iterable +from unittest.mock import MagicMock + +from _pyrepl.console import Console, Event +from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig +from _pyrepl.simple_interact import _strip_final_indent + + +def multiline_input(reader: ReadlineAlikeReader, namespace: dict | None = None): + saved = reader.more_lines + try: + reader.more_lines = partial(more_lines, namespace=namespace) + reader.ps1 = reader.ps2 = ">>>" + reader.ps3 = reader.ps4 = "..." + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + +def more_lines(text: str, namespace: dict | None = None): + if namespace is None: + namespace = {} + src = _strip_final_indent(text) + console = InteractiveConsole(namespace, filename="") + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + +def code_to_events(code: str): + for c in code: + yield Event(evt="key", data=c, raw=bytearray(c.encode("utf-8"))) + + +def prepare_reader(console: Console, **kwargs): + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + reader.more_lines = partial(more_lines, namespace=None) + reader.paste_mode = True # Avoid extra indents + + def get_prompt(lineno, cursor_on_line) -> str: + return "" + + reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) + + for key, val in kwargs.items(): + setattr(reader, key, val) + + return reader + + +def prepare_console(events: Iterable[Event], **kwargs): + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def handle_all_events( + events, prepare_console=prepare_console, prepare_reader=prepare_reader +): + console = prepare_console(events) + reader = prepare_reader(console) + try: + while True: + reader.handle1() + except StopIteration: + pass + return reader, console + + +handle_events_narrow_console = partial( + handle_all_events, + prepare_console=partial(prepare_console, width=10), +) + + +class FakeConsole(Console): + def __init__(self, events, encoding="utf-8"): + self.events = iter(events) + self.encoding = encoding + self.screen = [] + self.height = 100 + self.width = 80 + + def get_event(self, block: bool = True) -> Event | None: + return next(self.events) + + def getpending(self) -> Event: + return self.get_event(block=False) + + def getheightwidth(self) -> tuple[int, int]: + return self.height, self.width + + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: + pass + + def prepare(self) -> None: + pass + + def restore(self) -> None: + pass + + def move_cursor(self, x: int, y: int) -> None: + pass + + def set_cursor_vis(self, visible: bool) -> None: + pass + + def push_char(self, char: int | bytes) -> None: + pass + + def beep(self) -> None: + pass + + def clear(self) -> None: + pass + + def finish(self) -> None: + pass + + def flushoutput(self) -> None: + pass + + def forgetinput(self) -> None: + pass + + def wait(self) -> None: + pass + + def repaint(self) -> None: + pass diff --git a/Lib/test/test_pyrepl/test_input.py b/Lib/test/test_pyrepl/test_input.py new file mode 100644 index 00000000000000..c78c876c2c4c2a --- /dev/null +++ b/Lib/test/test_pyrepl/test_input.py @@ -0,0 +1,102 @@ +import unittest + +from _pyrepl.console import Event +from _pyrepl.input import KeymapTranslator + + +class KeymapTranslatorTests(unittest.TestCase): + def test_push_single_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_multiple_keys(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_invalid_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "b") + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["b"])) + + def test_push_invalid_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_character_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_character_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + evt3 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key_interrupted(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + evt3 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_invalid_key_with_unicode_category(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "\u0003") # Control character + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["\u0003"])) + + def test_empty(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + self.assertTrue(translator.empty()) + evt = Event("key", "a") + translator.push(evt) + self.assertFalse(translator.empty()) + translator.get() + self.assertTrue(translator.empty()) diff --git a/Lib/test/test_pyrepl/test_keymap.py b/Lib/test/test_pyrepl/test_keymap.py new file mode 100644 index 00000000000000..419f1643030b9d --- /dev/null +++ b/Lib/test/test_pyrepl/test_keymap.py @@ -0,0 +1,74 @@ +import unittest + +from _pyrepl.keymap import parse_keys, compile_keymap + + +class TestParseKeys(unittest.TestCase): + def test_single_character(self): + self.assertEqual(parse_keys("a"), ["a"]) + self.assertEqual(parse_keys("b"), ["b"]) + self.assertEqual(parse_keys("1"), ["1"]) + + def test_escape_sequences(self): + self.assertEqual(parse_keys("\\n"), ["\n"]) + self.assertEqual(parse_keys("\\t"), ["\t"]) + self.assertEqual(parse_keys("\\\\"), ["\\"]) + self.assertEqual(parse_keys("\\'"), ["'"]) + self.assertEqual(parse_keys('\\"'), ['"']) + + def test_control_sequences(self): + self.assertEqual(parse_keys("\\C-a"), ["\x01"]) + self.assertEqual(parse_keys("\\C-b"), ["\x02"]) + self.assertEqual(parse_keys("\\C-c"), ["\x03"]) + + def test_meta_sequences(self): + self.assertEqual(parse_keys("\\M-a"), ["\033", "a"]) + self.assertEqual(parse_keys("\\M-b"), ["\033", "b"]) + self.assertEqual(parse_keys("\\M-c"), ["\033", "c"]) + + def test_keynames(self): + self.assertEqual(parse_keys("\\"), ["up"]) + self.assertEqual(parse_keys("\\"), ["down"]) + self.assertEqual(parse_keys("\\"), ["left"]) + self.assertEqual(parse_keys("\\"), ["right"]) + + def test_combinations(self): + self.assertEqual(parse_keys("\\C-a\\n\\"), ["\x01", "\n", "up"]) + self.assertEqual(parse_keys("\\M-a\\t\\"), ["\033", "a", "\t", "down"]) + + +class TestCompileKeymap(unittest.TestCase): + def test_empty_keymap(self): + keymap = {} + result = compile_keymap(keymap) + self.assertEqual(result, {}) + + def test_single_keymap(self): + keymap = {b"a": "action"} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": "action"}) + + def test_nested_keymap(self): + keymap = {b"a": {b"b": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action"}}) + + def test_empty_value(self): + keymap = {b"a": {b"": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action"}}) + + def test_multiple_empty_values(self): + keymap = {b"a": {b"": "action1", b"b": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action1", b"b": "action2"}}) + + def test_multiple_keymaps(self): + keymap = {b"a": {b"b": "action1", b"c": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action1", b"c": "action2"}}) + + def test_nested_multiple_keymaps(self): + keymap = {b"a": {b"b": {b"c": "action"}}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": {b"c": "action"}}}) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py new file mode 100644 index 00000000000000..bc0a9975e34e00 --- /dev/null +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -0,0 +1,639 @@ +import itertools +import os +import rlcompleter +import unittest +from unittest import TestCase + +from .support import FakeConsole, handle_all_events, handle_events_narrow_console, multiline_input, code_to_events +from _pyrepl.console import Event +from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig + + +class TestCursorPosition(TestCase): + def test_up_arrow_simple(self): + # fmt: off + code = ( + 'def f():\n' + ' ...\n' + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 1)) + console.move_cursor.assert_called_once_with(0, 1) + + def test_down_arrow_end_of_input(self): + # fmt: off + code = ( + 'def f():\n' + ' ...\n' + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 2)) + console.move_cursor.assert_called_once_with(0, 2) + + def test_left_arrow_simple(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (4, 0)) + console.move_cursor.assert_called_once_with(4, 0) + + def test_right_arrow_end_of_line(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (5, 0)) + console.move_cursor.assert_called_once_with(5, 0) + + def test_cursor_position_simple_character(self): + events = itertools.chain(code_to_events("k")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 1 for simple character + self.assertEqual(reader.cxy, (1, 0)) + + def test_cursor_position_double_width_character(self): + events = itertools.chain(code_to_events("樂")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_character_move_left(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 0) + self.assertEqual(reader.cxy, (0, 0)) + + def test_cursor_position_double_width_character_move_left_right(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_characters_move_up(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor at end of first line + self.assertEqual(reader.pos, len(for_loop)) + self.assertEqual(reader.cxy, (len(for_loop), 0)) + + def test_cursor_position_double_width_characters_move_up_down(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor here (showing 2nd line only): + # < ' 可口可乐; 可口可樂'> + # ^ + self.assertEqual(reader.pos, 19) + self.assertEqual(reader.cxy, (10, 1)) + + def test_cursor_position_multiple_double_width_characters_move_left(self): + events = itertools.chain( + code_to_events("' 可口可乐; 可口可樂'"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 10) + + # 1 for quote, 1 for space, 2 per wide character, + # 1 for semicolon, 1 for space, 2 per wide character + self.assertEqual(reader.cxy, (16, 0)) + + def test_cursor_position_move_up_to_eol(self): + first_line = "for _ in _:" + second_line = " hello" + + # fmt: off + code = ( + f"{first_line}\n" + f"{second_line}\n" + " h\n" + " hel" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 1, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual( + reader.pos, len(first_line) + len(second_line) + 1 + ) # +1 for newline + self.assertEqual(reader.cxy, (len(second_line), 1)) + + def test_cursor_position_move_down_to_eol(self): + last_line = " hel" + + # fmt: off + code = ( + "for _ in _:\n" + " hello\n" + " h\n" + f"{last_line}" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 3, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual(reader.pos, len(code)) + self.assertEqual(reader.cxy, (len(last_line), 3)) + + def test_cursor_position_multiple_mixed_lines_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " x = '可口可乐; 可口可樂'\n" + " y = 'abckdfjskldfjslkdjf'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + 13 * [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + + reader, _ = handle_all_events(events) + + # By moving left, we're before the s: + # y = 'abckdfjskldfjslkdjf' + # ^ + # And we should move before the semi-colon despite the different offset + # x = '可口可乐; 可口可樂' + # ^ + self.assertEqual(reader.pos, 22) + self.assertEqual(reader.cxy, (15, 1)) + + def test_cursor_position_after_wrap_and_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + reader, _ = handle_events_narrow_console(events) + + # The code looks like this: + # def foo()\ + # : + # hello + # After moving up we should be after the colon in line 2 + self.assertEqual(reader.pos, 10) + self.assertEqual(reader.cxy, (1, 1)) + + +class TestPyReplOutput(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_basic(self): + reader = self.prepare_reader(code_to_events("1+1\n")) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_multiline_edit(self): + events = itertools.chain( + code_to_events("def f():\n ...\n\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "def f():\n ...\n ") + output = multiline_input(reader) + self.assertEqual(output, "def g():\n ...\n ") + + def test_history_navigation_with_up_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_history_navigation_with_down_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_history_search(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n3+3\n"), + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "3+3") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_control_character(self): + events = code_to_events("c\x1d\n") + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, "c\x1d") + + +class TestPyReplCompleter(TestCase): + def prepare_reader(self, events, namespace): + console = FakeConsole(events) + config = ReadlineConfig() + config.readline_completer = rlcompleter.Completer(namespace).complete + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_simple_completion(self): + events = code_to_events("os.geten\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.getenv") + + def test_completion_with_many_options(self): + # Test with something that initially displays many options + # and then complete from one of them. The first time tab is + # pressed, the options are displayed (which corresponds to + # when the repl shows [ not unique ]) and the second completes + # from one of them. + events = code_to_events("os.\t\tO_AP\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.O_APPEND") + + def test_empty_namespace_completion(self): + events = code_to_events("os.geten\t\n") + namespace = {} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.geten") + + def test_global_namespace_completion(self): + events = code_to_events("py\t\n") + namespace = {"python": None} + reader = self.prepare_reader(events, namespace) + output = multiline_input(reader, namespace) + self.assertEqual(output, "python") + + def test_updown_arrow_with_completion_menu(self): + """Up arrow in the middle of unfinished tab completion when the menu is displayed + should work and trigger going back in history. Down arrow should subsequently + get us back to the incomplete command.""" + code = "import os\nos.\t\t" + namespace = {"os": os} + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events, namespace=namespace) + output = multiline_input(reader, namespace) + # This is the first line, nothing to see here + self.assertEqual(output, "import os") + # This is the second line. We pressed up and down arrows + # so we should end up where we were when we initiated tab completion. + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.") + + +class TestPasteEvent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_paste(self): + # fmt: off + code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:\n' + ' pass\n' + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines(self): + # fmt: off + code = ( + 'def f():\n' + ' x = y\n' + ' \n' + ' y = z\n' + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines_not_in_paste_mode(self): + # fmt: off + code = ( + 'def f():\n' + ' x = y\n' + ' \n' + ' y = z\n\n' + ) + + expected = ( + 'def f():\n' + ' x = y\n' + ' ' + ) + # fmt: on + + events = code_to_events(code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, expected) + + def test_paste_not_in_paste_mode(self): + # fmt: off + input_code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:\n' + ' pass\n\n' + ) + + output_code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:' + ) + # fmt: on + + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste(self): + """Test that bracketed paste using \x1b[200~ and \x1b[201~ works.""" + # fmt: off + input_code = ( + 'def a():\n' + ' for x in range(10):\n' + '\n' + ' if x%2:\n' + ' print(x)\n' + '\n' + ' else:\n' + ' pass\n' + ) + + output_code = ( + 'def a():\n' + ' for x in range(10):\n' + '\n' + ' if x%2:\n' + ' print(x)\n' + '\n' + ' else:\n' + ' pass\n' + ) + # fmt: on + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste_single_line(self): + input_code = "oneline" + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, input_code) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py new file mode 100644 index 00000000000000..dc7d8a5ba97cda --- /dev/null +++ b/Lib/test/test_pyrepl/test_reader.py @@ -0,0 +1,135 @@ +import itertools +from unittest import TestCase + +from .support import handle_all_events, handle_events_narrow_console, code_to_events +from _pyrepl.console import Event + + +class TestReader(TestCase): + def assert_screen_equals(self, reader, expected): + actual = reader.calc_screen() + expected = expected.split("\n") + self.assertListEqual(actual, expected) + + def test_calc_screen_wrap_simple(self): + events = code_to_events(10 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_calc_screen_wrap_wide_characters(self): + events = code_to_events(8 * "a" + "樂") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{8*"a"}\\\n樂") + + def test_calc_screen_wrap_three_lines(self): + events = code_to_events(20 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") + + def test_calc_screen_wrap_three_lines_mixed_character(self): + # fmt: off + code = ( + "def f():\n" + f" {8*"a"}\n" + f" {5*"樂"}" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + + # fmt: off + self.assert_screen_equals(reader, ( + "def f():\n" + f" {7*"a"}\\\n" + "a\n" + f" {3*"樂"}\\\n" + "樂樂" + )) + # fmt: on + + def test_calc_screen_backspace(self): + events = itertools.chain( + code_to_events("aaa"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "aa") + + def test_calc_screen_wrap_removes_after_backspace(self): + events = itertools.chain( + code_to_events(10 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, 9 * "a") + + def test_calc_screen_backspace_in_second_line_after_wrap(self): + events = itertools.chain( + code_to_events(11 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_setpos_for_xy_simple(self): + events = code_to_events("11+11") + reader, _ = handle_all_events(events) + reader.setpos_from_xy(0, 0) + self.assertEqual(reader.pos, 0) + + def test_setpos_from_xy_multiple_lines(self): + # fmt: off + code = ( + "def foo():\n" + " return 1" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_all_events(events) + reader.setpos_from_xy(2, 1) + self.assertEqual(reader.pos, 13) + + def test_setpos_from_xy_after_wrap(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(2, 2) + self.assertEqual(reader.pos, 13) + + def test_setpos_fromxy_in_wrapped_line(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(0, 1) + self.assertEqual(reader.pos, 9) + + def test_up_arrow_after_ctrl_r(self): + events = iter( + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ] + ) + + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "") diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py new file mode 100644 index 00000000000000..cec3ae033325ac --- /dev/null +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -0,0 +1,294 @@ +import itertools +from functools import partial +from unittest import TestCase +from unittest.mock import MagicMock, call, patch, ANY + +from .support import handle_all_events, code_to_events +from _pyrepl.console import Event +from _pyrepl.unix_console import UnixConsole + + +def unix_console(events, **kwargs): + console = UnixConsole() + console.get_event = MagicMock(side_effect=events) + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +handle_events_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console), +) +handle_events_narrow_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, width=5), +) +handle_events_short_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, height=1), +) +handle_events_unix_console_height_3 = partial( + handle_all_events, prepare_console=partial(unix_console, height=3) +) + + +TERM_CAPABILITIES = { + "bel": b"\x07", + "civis": b"\x1b[?25l", + "clear": b"\x1b[H\x1b[2J", + "cnorm": b"\x1b[?12l\x1b[?25h", + "cub": b"\x1b[%p1%dD", + "cub1": b"\x08", + "cud": b"\x1b[%p1%dB", + "cud1": b"\n", + "cuf": b"\x1b[%p1%dC", + "cuf1": b"\x1b[C", + "cup": b"\x1b[%i%p1%d;%p2%dH", + "cuu": b"\x1b[%p1%dA", + "cuu1": b"\x1b[A", + "dch1": b"\x1b[P", + "dch": b"\x1b[%p1%dP", + "el": b"\x1b[K", + "hpa": b"\x1b[%i%p1%dG", + "ich": b"\x1b[%p1%d@", + "ich1": None, + "ind": b"\n", + "pad": None, + "ri": b"\x1bM", + "rmkx": b"\x1b[?1l\x1b>", + "smkx": b"\x1b[?1h\x1b=", +} + + +@patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) +@patch( + "_pyrepl.curses.tparm", + lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), +) +@patch("_pyrepl.curses.setupterm", lambda a, b: None) +@patch( + "termios.tcgetattr", + lambda _: [ + 27394, + 3, + 19200, + 536872399, + 38400, + 38400, + [ + b"\x04", + b"\xff", + b"\xff", + b"\x7f", + b"\x17", + b"\x15", + b"\x12", + b"\x00", + b"\x03", + b"\x1c", + b"\x1a", + b"\x19", + b"\x11", + b"\x13", + b"\x16", + b"\x0f", + b"\x01", + b"\x00", + b"\x14", + b"\x00", + ], + ], +) +@patch("termios.tcsetattr", lambda a, b, c: None) +@patch("os.write") +class TestConsole(TestCase): + def test_simple_addition(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"4") + + def test_wrap(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, _ = handle_events_narrow_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"\\") + _os_write.assert_any_call(ANY, b"\n") + _os_write.assert_any_call(ANY, b"4") + + def test_cursor_left(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + + def test_cursor_left_right(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuf"] + b":1") + + def test_cursor_up(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + + def test_cursor_up_down(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cud"] + b":1") + + def test_cursor_back_write(self, _os_write): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, _ = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, b"2") + + def test_multiline_function_move_up_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, _ = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + + def test_multiline_function_move_up_down_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, _ = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ind"] + b":") + + def test_resize_bigger_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_short_unix_console(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, _ = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ri"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b"def f():"), + ] + ) + + def test_resize_smaller_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_unix_console_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, _ = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ind"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b" foo"), + ] + ) diff --git a/Lib/test/test_pyrepl/test_unix_eventqueue.py b/Lib/test/test_pyrepl/test_unix_eventqueue.py new file mode 100644 index 00000000000000..be2bc00e5692a7 --- /dev/null +++ b/Lib/test/test_pyrepl/test_unix_eventqueue.py @@ -0,0 +1,113 @@ +import tempfile +import unittest +from unittest.mock import patch + +from _pyrepl.console import Event +from _pyrepl.unix_eventqueue import EventQueue + + +@patch("_pyrepl.curses.tigetstr", lambda x: b"") +class TestUnivEventQueue(unittest.TestCase): + def setUp(self): + self.file = tempfile.TemporaryFile() + + def tearDown(self) -> None: + self.file.close() + + def test_get(self): + eq = EventQueue(self.file.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.get(), event) + + def test_empty(self): + eq = EventQueue(self.file.fileno(), "utf-8") + self.assertTrue(eq.empty()) + eq.insert(Event("key", "a", b"a")) + self.assertFalse(eq.empty()) + + def test_flush_buf(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.buf.extend(b"test") + self.assertEqual(eq.flush_buf(), b"test") + self.assertEqual(eq.buf, bytearray()) + + def test_insert(self): + eq = EventQueue(self.file.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.events[0], event) + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": "b"} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "b") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_without_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"c": "d"} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "a") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.push("b") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "c") + eq.push("d") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "d") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.flush_buf() + eq.push("\033") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\033") + eq.push("b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "b") + + def test_push_special_key(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("A") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + + def test_push_unrecognized_escape_sequence(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("Z") + self.assertEqual(len(eq.events), 3) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "[") + self.assertEqual(eq.events[2].evt, "key") + self.assertEqual(eq.events[2].data, "Z") diff --git a/Makefile.pre.in b/Makefile.pre.in index 74a438b015a97d..9e99c95e2af042 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2447,6 +2447,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_pathlib \ test/test_peg_generator \ test/test_pydoc \ + test/test_pyrepl \ test/test_sqlite3 \ test/test_tkinter \ test/test_tomllib \ From 77ff28bb6776d583e593937df554cbf572cb47b0 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 21 May 2024 13:08:51 -0400 Subject: [PATCH 146/903] gh-109176: replace _PyFrame_OpAlreadyRan by an assertion that the frame is complete. (#119234) --- Objects/frameobject.c | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 64fded85de1468..fc8d6c7a7aee89 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1828,32 +1828,6 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, return f; } -static int -_PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) -{ - // This only works when opcode is a non-quickened form: - assert(_PyOpcode_Deopt[opcode] == opcode); - int check_oparg = 0; - for (_Py_CODEUNIT *instruction = _PyCode_CODE(_PyFrame_GetCode(frame)); - instruction < frame->instr_ptr; instruction++) - { - int check_opcode = _PyOpcode_Deopt[instruction->op.code]; - check_oparg |= instruction->op.arg; - if (check_opcode == opcode && check_oparg == oparg) { - return 1; - } - if (check_opcode == EXTENDED_ARG) { - check_oparg <<= 8; - } - else { - check_oparg = 0; - } - instruction += _PyOpcode_Caches[check_opcode]; - } - return 0; -} - - // Initialize frame free variables if needed static void frame_init_get_vars(_PyInterpreterFrame *frame) @@ -1907,14 +1881,9 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, value = PyCell_GET(value); } else if (kind & CO_FAST_CELL) { - // Note that no *_DEREF ops can happen before MAKE_CELL - // executes. So there's no need to duplicate the work - // that MAKE_CELL would otherwise do later, if it hasn't - // run yet. if (value != NULL) { - if (PyCell_Check(value) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. + if (PyCell_Check(value)) { + assert(!_PyFrame_IsIncomplete(frame)); value = PyCell_GET(value); } // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), From 87939bd5790accea77c5a81093f16f28d3f0b429 Mon Sep 17 00:00:00 2001 From: Arnon Yaari Date: Tue, 21 May 2024 13:16:34 -0400 Subject: [PATCH 147/903] gh-117657: Fix itertools.count thread safety (#119268) Fix itertools.count in free-threading mode --- Lib/test/test_itertools.py | 24 ++++++++++++- Modules/itertoolsmodule.c | 40 +++++++++++++++++----- Tools/tsan/suppressions_free_threading.txt | 1 - 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 4d2c01886724a8..53b8064c3cfe82 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -546,7 +546,7 @@ def test_count(self): #check proper internal error handling for large "step' sizes count(1, maxsize+5); sys.exc_info() - def test_count_with_stride(self): + def test_count_with_step(self): self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) self.assertEqual(lzip('abc',count(start=2,step=3)), [('a', 2), ('b', 5), ('c', 8)]) @@ -590,6 +590,28 @@ def test_count_with_stride(self): self.assertEqual(type(next(c)), int) self.assertEqual(type(next(c)), float) + @threading_helper.requires_working_threading() + def test_count_threading(self, step=1): + # this test verifies multithreading consistency, which is + # mostly for testing builds without GIL, but nice to test anyway + count_to = 10_000 + num_threads = 10 + c = count(step=step) + def counting_thread(): + for i in range(count_to): + next(c) + threads = [] + for i in range(num_threads): + thread = threading.Thread(target=counting_thread) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + self.assertEqual(next(c), count_to * num_threads * step) + + def test_count_with_step_threading(self): + self.test_count_threading(step=5) + def test_cycle(self): self.assertEqual(take(10, cycle('abc')), list('abcabcabca')) self.assertEqual(list(cycle('')), []) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ae316d9e369d45..e740ec4d7625c3 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,13 +1,14 @@ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_tuple.h" // _PyTuple_ITEMS() -#include // offsetof() +#include // offsetof() /* Itertools module written and maintained by Raymond D. Hettinger @@ -3254,7 +3255,7 @@ fast_mode: when cnt an integer < PY_SSIZE_T_MAX and no step is specified. assert(cnt != PY_SSIZE_T_MAX && long_cnt == NULL && long_step==PyLong(1)); Advances with: cnt += 1 - When count hits Y_SSIZE_T_MAX, switch to slow_mode. + When count hits PY_SSIZE_T_MAX, switch to slow_mode. slow_mode: when cnt == PY_SSIZE_T_MAX, step is not int(1), or cnt is a float. @@ -3403,9 +3404,30 @@ count_nextlong(countobject *lz) static PyObject * count_next(countobject *lz) { +#ifndef Py_GIL_DISABLED if (lz->cnt == PY_SSIZE_T_MAX) return count_nextlong(lz); return PyLong_FromSsize_t(lz->cnt++); +#else + // free-threading version + // fast mode uses compare-exchange loop + // slow mode uses a critical section + PyObject *returned; + Py_ssize_t cnt; + + cnt = _Py_atomic_load_ssize_relaxed(&lz->cnt); + for (;;) { + if (cnt == PY_SSIZE_T_MAX) { + Py_BEGIN_CRITICAL_SECTION(lz); + returned = count_nextlong(lz); + Py_END_CRITICAL_SECTION(); + return returned; + } + if (_Py_atomic_compare_exchange_ssize(&lz->cnt, &cnt, cnt + 1)) { + return PyLong_FromSsize_t(cnt); + } + } +#endif } static PyObject * diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index dfa4a1fe9ca438..cda57d78067bb3 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -56,7 +56,6 @@ race_top:_Py_dict_lookup_threadsafe race_top:_imp_release_lock race_top:_multiprocessing_SemLock_acquire_impl race_top:builtin_compile_impl -race_top:count_next race_top:dictiter_new race_top:dictresize race_top:insert_to_emptydict From f6da790122fdae1a28f444edfbb55202d6829cd1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 21 May 2024 19:51:51 +0200 Subject: [PATCH 148/903] gh-111389: Add PyHASH_MULTIPLIER constant (#119214) --- Doc/c-api/hash.rst | 6 ++++++ Include/cpython/pyhash.h | 5 +++-- .../C API/2024-05-20-10-35-22.gh-issue-111389.a6axBk.rst | 2 ++ Objects/codeobject.c | 2 +- Python/optimizer.c | 2 +- Python/pyhash.c | 4 ++-- Python/tracemalloc.c | 2 +- 7 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-05-20-10-35-22.gh-issue-111389.a6axBk.rst diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index ddf0b3e15dbdbe..7345a048a4128b 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -29,6 +29,12 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. versionadded:: 3.13 +.. c:macro:: PyHASH_MULTIPLIER + + Prime multiplier used in string and various other hashes. + + .. versionadded:: 3.13 + .. c:macro:: PyHASH_INF The hash value returned for a positive infinity. diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index 2f8e12c1423aa1..825c034a8d8474 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -3,7 +3,7 @@ #endif /* Prime multiplier used in string and various other hashes. */ -#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ +#define PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ /* Parameters used for the numeric hash implementation. See notes for _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on @@ -17,9 +17,10 @@ #define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) #define PyHASH_INF 314159 -#define PyHASH_IMAG _PyHASH_MULTIPLIER +#define PyHASH_IMAG PyHASH_MULTIPLIER /* Aliases kept for backward compatibility with Python 3.12 */ +#define _PyHASH_MULTIPLIER PyHASH_MULTIPLIER #define _PyHASH_BITS PyHASH_BITS #define _PyHASH_MODULUS PyHASH_MODULUS #define _PyHASH_INF PyHASH_INF diff --git a/Misc/NEWS.d/next/C API/2024-05-20-10-35-22.gh-issue-111389.a6axBk.rst b/Misc/NEWS.d/next/C API/2024-05-20-10-35-22.gh-issue-111389.a6axBk.rst new file mode 100644 index 00000000000000..f47662f2e0a778 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-20-10-35-22.gh-issue-111389.a6axBk.rst @@ -0,0 +1,2 @@ +Add :c:macro:`PyHASH_MULTIPLIER` constant: prime multiplier used in string +and various other hashes. Patch by Victor Stinner. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 3d804f73a54088..1c6b55643fa19e 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2020,7 +2020,7 @@ code_hash(PyCodeObject *co) Py_uhash_t uhash = 20221211; #define SCRAMBLE_IN(H) do { \ uhash ^= (Py_uhash_t)(H); \ - uhash *= _PyHASH_MULTIPLIER; \ + uhash *= PyHASH_MULTIPLIER; \ } while (0) #define SCRAMBLE_IN_HASH(EXPR) do { \ Py_hash_t h = PyObject_Hash(EXPR); \ diff --git a/Python/optimizer.c b/Python/optimizer.c index 9ae99ccdaea2e7..5b4a6ff8cb3dad 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1496,7 +1496,7 @@ address_to_hash(void *ptr) { uintptr_t addr = (uintptr_t)ptr; for (int i = 0; i < SIZEOF_VOID_P; i++) { uhash ^= addr & 255; - uhash *= (uint64_t)_PyHASH_MULTIPLIER; + uhash *= (uint64_t)PyHASH_MULTIPLIER; addr >>= 8; } return uhash; diff --git a/Python/pyhash.c b/Python/pyhash.c index d508d78092a9e7..5263622ff3126d 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -263,12 +263,12 @@ fnv(const void *src, Py_ssize_t len) x ^= (Py_uhash_t) *p << 7; while (blocks--) { PY_UHASH_CPY(block.bytes, p); - x = (_PyHASH_MULTIPLIER * x) ^ block.value; + x = (PyHASH_MULTIPLIER * x) ^ block.value; p += SIZEOF_PY_UHASH_T; } /* add remainder */ for (; remainder > 0; remainder--) - x = (_PyHASH_MULTIPLIER * x) ^ (Py_uhash_t) *p++; + x = (PyHASH_MULTIPLIER * x) ^ (Py_uhash_t) *p++; x ^= (Py_uhash_t) len; x ^= (Py_uhash_t) _Py_HashSecret.fnv.suffix; if (x == (Py_uhash_t) -1) { diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index e3ec72062f6931..fee7dd0e56d96d 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -312,7 +312,7 @@ traceback_hash(traceback_t *traceback) /* code based on tuplehash() of Objects/tupleobject.c */ Py_uhash_t x, y; /* Unsigned for defined overflow behavior. */ int len = traceback->nframe; - Py_uhash_t mult = _PyHASH_MULTIPLIER; + Py_uhash_t mult = PyHASH_MULTIPLIER; frame_t *frame; x = 0x345678UL; From b7f45a9332e8a2b78c2ccee79e5f251864363be5 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Wed, 22 May 2024 02:35:05 +0800 Subject: [PATCH 149/903] Fix typos in documentation (#119295) --- Misc/NEWS.d/3.12.0a1.rst | 6 +++--- Misc/NEWS.d/3.12.0a2.rst | 2 +- Misc/NEWS.d/3.12.0a3.rst | 2 +- Misc/NEWS.d/3.12.0a4.rst | 2 +- Misc/NEWS.d/3.12.0a5.rst | 2 +- Misc/NEWS.d/3.12.0a6.rst | 2 +- Misc/NEWS.d/3.12.0b1.rst | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 1f259a64ee4494..84d9d4e017609d 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -560,7 +560,7 @@ versions prior to 3.11 .. nonce: 9lmTCC .. section: Core and Builtins -Remove two cases of undefined behavoir, by adding NULL checks. +Remove two cases of undefined behavior, by adding NULL checks. .. @@ -2591,7 +2591,7 @@ Update bundled pip to 22.2.2. Fix :class:`asyncio.TaskGroup` to propagate exception when :exc:`asyncio.CancelledError` was replaced with another exception by a -context manger. Patch by Kumar Aditya and Guido van Rossum. +context manager. Patch by Kumar Aditya and Guido van Rossum. .. @@ -4476,7 +4476,7 @@ they are deprecated. Contributed by C.A.M. Gerlach. .. nonce: we7AFm .. section: Documentation -Replaced incorrectly written true/false values in documentiation. Patch by +Replaced incorrectly written true/false values in documentation. Patch by Robert O'Shea .. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index f1d69d9b3e7638..88d84ad93b35b5 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -822,7 +822,7 @@ and to indicate when it became late-bound. .. nonce: 7KinCV .. section: Tests -The Python test suite now fails wit exit code 4 if no tests ran. It should +The Python test suite now fails with exit code 4 if no tests ran. It should help detecting typos in test names and test methods. .. diff --git a/Misc/NEWS.d/3.12.0a3.rst b/Misc/NEWS.d/3.12.0a3.rst index 9b789c68607c73..07593998d80891 100644 --- a/Misc/NEWS.d/3.12.0a3.rst +++ b/Misc/NEWS.d/3.12.0a3.rst @@ -82,7 +82,7 @@ Victor Stinner. .. section: Core and Builtins Fixed a bug that was causing a buffer overflow if the tokenizer copies a -line missing the newline caracter from a file that is as long as the +line missing the newline character from a file that is as long as the available tokenizer buffer. Patch by Pablo galindo .. diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index 1599084ae0d323..d7af30f6c09b2b 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -830,7 +830,7 @@ Reduced the memory usage of :func:`urllib.parse.unquote` and .. section: Library ``inspect.signature`` was raising ``TypeError`` on call with mock objects. -Now it correctly returns ``(*args, **kwargs)`` as infered signature. +Now it correctly returns ``(*args, **kwargs)`` as inferred signature. .. diff --git a/Misc/NEWS.d/3.12.0a5.rst b/Misc/NEWS.d/3.12.0a5.rst index 8cf90b0e9cde46..effda2be6fd26c 100644 --- a/Misc/NEWS.d/3.12.0a5.rst +++ b/Misc/NEWS.d/3.12.0a5.rst @@ -506,7 +506,7 @@ inheritance. .. nonce: 7sQz5l .. section: Build -Update BOLT configration not to use depreacted usage of ``--split +Update BOLT configuration not to use deprecated usage of ``--split functions``. Patch by Donghee Na. .. diff --git a/Misc/NEWS.d/3.12.0a6.rst b/Misc/NEWS.d/3.12.0a6.rst index 05f9243eb6b1bc..382dae33fcaee1 100644 --- a/Misc/NEWS.d/3.12.0a6.rst +++ b/Misc/NEWS.d/3.12.0a6.rst @@ -453,7 +453,7 @@ E. Aasland. .. section: Library Change repr of :class:`collections.OrderedDict` to use regular dictionary -formating instead of pairs of keys and values. +formatting instead of pairs of keys and values. .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index a264f7fad9e65a..9f3095b224233e 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -1446,7 +1446,7 @@ Adapt the :mod:`winsound` extension module to :pep:`687`. .. nonce: jurMzv .. section: Library -Remove deprecation of enum ``memmber.member`` access. +Remove deprecation of enum ``member.member`` access. .. From 10b1bd926a5546e0f5cbd1a47d00dc5ff84f1979 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 21 May 2024 22:28:05 +0300 Subject: [PATCH 150/903] gh-119189: Add yet more tests for mixed Fraction arithmetic (GH-119298) --- Lib/test/test_fractions.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 3a9a86fe7a8b67..3a714c64278847 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -135,6 +135,14 @@ def __str__(self): def __repr__(self): return f'{self.__class__.__name__}({self.value!r})' +class SymbolicReal(Symbolic): + pass +numbers.Real.register(SymbolicReal) + +class SymbolicComplex(Symbolic): + pass +numbers.Complex.register(SymbolicComplex) + class Rat: """Simple Rational class for testing mixed arithmetic.""" def __init__(self, n, d): @@ -273,6 +281,8 @@ def __repr__(self): return f'{self.__class__.__name__}({self.x!r}, {self.y!r})' numbers.Complex.register(Rect) +class RectComplex(Rect, complex): + pass class FractionTest(unittest.TestCase): @@ -790,12 +800,17 @@ def testMixedMultiplication(self): self.assertTypedEquals(F(3, 2) * Root(4), Root(F(9, 1))) self.assertTypedEquals(Root(4) * F(3, 2), 3.0) + self.assertEqual(F(3, 2) * SymbolicReal('X'), SymbolicReal('3/2 * X')) + self.assertRaises(TypeError, operator.mul, SymbolicReal('X'), F(3, 2)) self.assertTypedEquals(F(3, 2) * Polar(4, 2), Polar(F(6, 1), 2)) self.assertTypedEquals(F(3, 2) * Polar(4.0, 2), Polar(6.0, 2)) self.assertTypedEquals(F(3, 2) * Rect(4, 3), Rect(F(6, 1), F(9, 2))) + self.assertTypedEquals(F(3, 2) * RectComplex(4, 3), RectComplex(6.0+0j, 4.5+0j)) self.assertRaises(TypeError, operator.mul, Polar(4, 2), F(3, 2)) self.assertTypedEquals(Rect(4, 3) * F(3, 2), 6.0 + 4.5j) + self.assertEqual(F(3, 2) * SymbolicComplex('X'), SymbolicComplex('3/2 * X')) + self.assertRaises(TypeError, operator.mul, SymbolicComplex('X'), F(3, 2)) self.assertEqual(F(3, 2) * Symbolic('X'), Symbolic('3/2 * X')) self.assertRaises(TypeError, operator.mul, Symbolic('X'), F(3, 2)) @@ -815,12 +830,16 @@ def testMixedDivision(self): self.assertTypedEquals(F(2, 3) / Root(4), Root(F(1, 9))) self.assertTypedEquals(Root(4) / F(2, 3), 3.0) + self.assertEqual(F(3, 2) / SymbolicReal('X'), SymbolicReal('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, SymbolicReal('X'), F(3, 2)) self.assertTypedEquals(F(3, 2) / Polar(4, 2), Polar(F(3, 8), -2)) self.assertTypedEquals(F(3, 2) / Polar(4.0, 2), Polar(0.375, -2)) self.assertTypedEquals(F(3, 2) / Rect(4, 3), Rect(0.24, 0.18)) self.assertRaises(TypeError, operator.truediv, Polar(4, 2), F(2, 3)) self.assertTypedEquals(Rect(4, 3) / F(2, 3), 6.0 + 4.5j) + self.assertEqual(F(3, 2) / SymbolicComplex('X'), SymbolicComplex('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, SymbolicComplex('X'), F(3, 2)) self.assertEqual(F(3, 2) / Symbolic('X'), Symbolic('3/2 / X')) self.assertRaises(TypeError, operator.truediv, Symbolic('X'), F(2, 3)) @@ -857,9 +876,14 @@ def testMixedIntegerDivision(self): self.assertRaises(TypeError, operator.mod, F(2, 3), Root(4)) self.assertTypedEquals(Root(4) % F(3, 2), 0.5) + self.assertEqual(F(3, 2) % SymbolicReal('X'), SymbolicReal('3/2 % X')) + self.assertRaises(TypeError, operator.mod, SymbolicReal('X'), F(3, 2)) self.assertRaises(TypeError, operator.mod, F(3, 2), Polar(4, 2)) + self.assertRaises(TypeError, operator.mod, F(3, 2), RectComplex(4, 3)) self.assertRaises(TypeError, operator.mod, Rect(4, 3), F(2, 3)) + self.assertEqual(F(3, 2) % SymbolicComplex('X'), SymbolicComplex('3/2 % X')) + self.assertRaises(TypeError, operator.mod, SymbolicComplex('X'), F(3, 2)) self.assertEqual(F(3, 2) % Symbolic('X'), Symbolic('3/2 % X')) self.assertRaises(TypeError, operator.mod, Symbolic('X'), F(2, 3)) @@ -888,7 +912,6 @@ def testMixedPower(self): self.assertIsInstance(F(4, 9) ** Rat(-3, 2), float) self.assertAlmostEqual(F(4, 9) ** Rat(-3, 2), 3.375) self.assertAlmostEqual(F(-4, 9) ** Rat(-3, 2), 3.375j) - self.assertTypedEquals(Rat(9, 4) ** F(3, 2), 3.375) self.assertTypedEquals(Rat(3, 2) ** F(3, 1), Rat(27, 8)) self.assertTypedEquals(Rat(3, 2) ** F(-3, 1), F(8, 27)) @@ -899,16 +922,22 @@ def testMixedPower(self): self.assertTypedEquals(Root(4) ** F(2, 1), Root(4, F(1))) self.assertTypedEquals(Root(4) ** F(-2, 1), Root(4, -F(1))) self.assertTypedEquals(Root(4) ** F(-2, 3), Root(4, -3.0)) + self.assertEqual(F(3, 2) ** SymbolicReal('X'), SymbolicReal('1.5 ** X')) + self.assertEqual(SymbolicReal('X') ** F(3, 2), SymbolicReal('X ** 1.5')) self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(2.25, 0.0)) self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(1.0, 0.0)) + self.assertTypedEquals(F(3, 2) ** RectComplex(2, 0), Polar(2.25, 0.0)) + self.assertTypedEquals(F(1, 1) ** RectComplex(2, 3), Polar(1.0, 0.0)) self.assertTypedEquals(Polar(4, 2) ** F(3, 2), Polar(8.0, 3.0)) self.assertTypedEquals(Polar(4, 2) ** F(3, 1), Polar(64, 6)) self.assertTypedEquals(Polar(4, 2) ** F(-3, 1), Polar(0.015625, -6)) self.assertTypedEquals(Polar(4, 2) ** F(-3, 2), Polar(0.125, -3.0)) + self.assertEqual(F(3, 2) ** SymbolicComplex('X'), SymbolicComplex('1.5 ** X')) + self.assertEqual(SymbolicComplex('X') ** F(3, 2), SymbolicComplex('X ** 1.5')) - self.assertTypedEquals(F(3, 2) ** Symbolic('X'), Symbolic('1.5 ** X')) - self.assertTypedEquals(Symbolic('X') ** F(3, 2), Symbolic('X ** 1.5')) + self.assertEqual(F(3, 2) ** Symbolic('X'), Symbolic('1.5 ** X')) + self.assertEqual(Symbolic('X') ** F(3, 2), Symbolic('X ** 1.5')) def testMixingWithDecimal(self): # Decimal refuses mixed arithmetic (but not mixed comparisons) From e3ed574f6acddf76e96a9b0fcbad35f15508cba1 Mon Sep 17 00:00:00 2001 From: Josh Cannon Date: Tue, 21 May 2024 14:37:32 -0500 Subject: [PATCH 151/903] gh-90562: Mention slots pitfall in dataclass docs (#107391) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Jelle Zijlstra Co-authored-by: Erlend E. Aasland Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/dataclasses.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 045bf6277289d8..cf707ca5b6802d 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -185,7 +185,10 @@ Module contents - *slots*: If true (the default is ``False``), :attr:`~object.__slots__` attribute will be generated and new class will be returned instead of the original one. If :attr:`!__slots__` is already defined in the class, then :exc:`TypeError` - is raised. + is raised. Calling no-arg :func:`super` in dataclasses using ``slots=True`` will result in + the following exception being raised: + ``TypeError: super(type, obj): obj must be an instance or subtype of type``. + The two-arg :func:`super` is a valid workaround. See :gh:`90562` for full details. .. versionadded:: 3.10 From 561ff1fa710493dee8c6482f990bd17535b27040 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 21 May 2024 16:30:45 -0400 Subject: [PATCH 152/903] gh-111201: Remove readline dependency from the PyREPL (#119262) --- Lib/_pyrepl/readline.py | 4 ++-- Lib/test/test_pyrepl/__init__.py | 1 - .../Library/2024-05-20-20-30-57.gh-issue-111201.DAA5lC.rst | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-20-20-30-57.gh-issue-111201.DAA5lC.rst diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 0adecf235a4eb4..9c85ce175fd6d6 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -31,9 +31,9 @@ from dataclasses import dataclass, field import os -import readline from site import gethistoryfile # type: ignore[attr-defined] import sys +from rlcompleter import Completer as RLCompleter from . import commands, historical_reader from .completing_reader import CompletingReader @@ -84,7 +84,7 @@ @dataclass class ReadlineConfig: - readline_completer: Completer | None = readline.get_completer() + readline_completer: Completer | None = RLCompleter().complete completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py index a9bc41f4d39f60..fa38b86b847dd9 100644 --- a/Lib/test/test_pyrepl/__init__.py +++ b/Lib/test/test_pyrepl/__init__.py @@ -7,7 +7,6 @@ # option. Additionally, we need to attempt to import curses and readline. requires("curses") curses = import_module("curses") -readline = import_module("readline") def load_tests(*args): diff --git a/Misc/NEWS.d/next/Library/2024-05-20-20-30-57.gh-issue-111201.DAA5lC.rst b/Misc/NEWS.d/next/Library/2024-05-20-20-30-57.gh-issue-111201.DAA5lC.rst new file mode 100644 index 00000000000000..15cd79dec378ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-20-20-30-57.gh-issue-111201.DAA5lC.rst @@ -0,0 +1 @@ +Remove dependency to :mod:`readline` from the new Python REPL. From de8f530841b55885b919677a6938ab33d4a92f20 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 21 May 2024 22:33:52 +0200 Subject: [PATCH 153/903] gh-119102: Fix REPL for dumb terminal (#119332) The site module gets the __main__ module to get _pyrepl.__main__. --- Lib/site.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index 4ba078388a37b8..f1a6d9cf66fdc3 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -523,7 +523,12 @@ def register_readline(): pass def write_history(): - from _pyrepl.__main__ import CAN_USE_PYREPL + try: + # _pyrepl.__main__ is executed as the __main__ module + from __main__ import CAN_USE_PYREPL + except ImportError: + CAN_USE_PYREPL = False + try: if os.getenv("PYTHON_BASIC_REPL") or not CAN_USE_PYREPL: readline.write_history_file(history) From b64182550f73e556344bd754d32e3be5d22a74e1 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 21 May 2024 22:36:36 +0200 Subject: [PATCH 154/903] gh-118507 : Refactor `nt._path_is*` to improve applicability for other cases (GH-118755) --- Lib/ntpath.py | 21 +- Lib/test/test_genericpath.py | 16 +- Lib/test/test_ntpath.py | 25 + ...-05-08-18-33-07.gh-issue-118507.OCQsAY.rst | 1 + Modules/clinic/posixmodule.c.h | 156 ++++- Modules/posixmodule.c | 599 +++++++++--------- 6 files changed, 453 insertions(+), 365 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-08-18-33-07.gh-issue-118507.OCQsAY.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index b833e0bad2645f..8d972cd1d0eb72 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -288,21 +288,6 @@ def dirname(p): return split(p)[0] -# Is a path a junction? - -if hasattr(os.stat_result, 'st_reparse_tag'): - def isjunction(path): - """Test whether a path is a junction""" - try: - st = os.lstat(path) - except (OSError, ValueError, AttributeError): - return False - return st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT -else: - # Use genericpath.isjunction as imported above - pass - - # Is a path a mount point? # Any drive letter root (eg c:\) # Any share UNC (eg \\server\share) @@ -911,13 +896,15 @@ def commonpath(paths): try: - # The isdir(), isfile(), islink() and exists() implementations in - # genericpath use os.stat(). This is overkill on Windows. Use simpler + # The isdir(), isfile(), islink(), exists() and lexists() implementations + # in genericpath use os.stat(). This is overkill on Windows. Use simpler # builtin functions if they are available. from nt import _path_isdir as isdir from nt import _path_isfile as isfile from nt import _path_islink as islink + from nt import _path_isjunction as isjunction from nt import _path_exists as exists + from nt import _path_lexists as lexists except ImportError: # Use genericpath.* as imported above pass diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index f407ee3caf154c..bf04b3fecf7057 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -135,6 +135,9 @@ def test_exists(self): self.assertIs(self.pathmodule.exists(filename), False) self.assertIs(self.pathmodule.exists(bfilename), False) + self.assertIs(self.pathmodule.lexists(filename), False) + self.assertIs(self.pathmodule.lexists(bfilename), False) + create_file(filename) self.assertIs(self.pathmodule.exists(filename), True) @@ -145,14 +148,13 @@ def test_exists(self): self.assertIs(self.pathmodule.exists(filename + '\x00'), False) self.assertIs(self.pathmodule.exists(bfilename + b'\x00'), False) - if self.pathmodule is not genericpath: - self.assertIs(self.pathmodule.lexists(filename), True) - self.assertIs(self.pathmodule.lexists(bfilename), True) + self.assertIs(self.pathmodule.lexists(filename), True) + self.assertIs(self.pathmodule.lexists(bfilename), True) - self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False) - self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False) - self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) - self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) + self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False) + self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False) + self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) + self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") @unittest.skipIf(is_emscripten, "Emscripten pipe fds have no stat") diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 7f91bf1c2b837a..9aa116682f7480 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1095,6 +1095,27 @@ def test_isfile_driveletter(self): raise unittest.SkipTest('SystemDrive is not defined or malformed') self.assertFalse(os.path.isfile('\\\\.\\' + drive)) + @unittest.skipUnless(hasattr(os, 'pipe'), "need os.pipe()") + def test_isfile_anonymous_pipe(self): + pr, pw = os.pipe() + try: + self.assertFalse(ntpath.isfile(pr)) + finally: + os.close(pr) + os.close(pw) + + @unittest.skipIf(sys.platform != 'win32', "windows only") + def test_isfile_named_pipe(self): + import _winapi + named_pipe = f'//./PIPE/python_isfile_test_{os.getpid()}' + h = _winapi.CreateNamedPipe(named_pipe, + _winapi.PIPE_ACCESS_INBOUND, + 0, 1, 0, 0, 0, 0) + try: + self.assertFalse(ntpath.isfile(named_pipe)) + finally: + _winapi.CloseHandle(h) + @unittest.skipIf(sys.platform != 'win32', "windows only") def test_con_device(self): self.assertFalse(os.path.isfile(r"\\.\CON")) @@ -1114,8 +1135,12 @@ def test_fast_paths_in_use(self): self.assertFalse(inspect.isfunction(os.path.isfile)) self.assertTrue(os.path.islink is nt._path_islink) self.assertFalse(inspect.isfunction(os.path.islink)) + self.assertTrue(os.path.isjunction is nt._path_isjunction) + self.assertFalse(inspect.isfunction(os.path.isjunction)) self.assertTrue(os.path.exists is nt._path_exists) self.assertFalse(inspect.isfunction(os.path.exists)) + self.assertTrue(os.path.lexists is nt._path_lexists) + self.assertFalse(inspect.isfunction(os.path.lexists)) @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") def test_isdevdrive(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-08-18-33-07.gh-issue-118507.OCQsAY.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-08-18-33-07.gh-issue-118507.OCQsAY.rst new file mode 100644 index 00000000000000..de1462f0d24fce --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-08-18-33-07.gh-issue-118507.OCQsAY.rst @@ -0,0 +1 @@ +Speedup :func:`os.path.isjunction` and :func:`os.path.lexists` on Windows with a native implementation. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index a0d1f3238a6733..5ec5635bae3f41 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2014,6 +2014,70 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #if defined(MS_WINDOWS) +PyDoc_STRVAR(os__path_exists__doc__, +"_path_exists($module, path, /)\n" +"--\n" +"\n" +"Test whether a path exists. Returns False for broken symbolic links."); + +#define OS__PATH_EXISTS_METHODDEF \ + {"_path_exists", (PyCFunction)os__path_exists, METH_O, os__path_exists__doc__}, + +static int +os__path_exists_impl(PyObject *module, PyObject *path); + +static PyObject * +os__path_exists(PyObject *module, PyObject *path) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = os__path_exists_impl(module, path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os__path_lexists__doc__, +"_path_lexists($module, path, /)\n" +"--\n" +"\n" +"Test whether a path exists. Returns True for broken symbolic links."); + +#define OS__PATH_LEXISTS_METHODDEF \ + {"_path_lexists", (PyCFunction)os__path_lexists, METH_O, os__path_lexists__doc__}, + +static int +os__path_lexists_impl(PyObject *module, PyObject *path); + +static PyObject * +os__path_lexists(PyObject *module, PyObject *path) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = os__path_lexists_impl(module, path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + PyDoc_STRVAR(os__path_isdir__doc__, "_path_isdir($module, /, s)\n" "--\n" @@ -2023,8 +2087,8 @@ PyDoc_STRVAR(os__path_isdir__doc__, #define OS__PATH_ISDIR_METHODDEF \ {"_path_isdir", _PyCFunction_CAST(os__path_isdir), METH_FASTCALL|METH_KEYWORDS, os__path_isdir__doc__}, -static PyObject * -os__path_isdir_impl(PyObject *module, PyObject *s); +static int +os__path_isdir_impl(PyObject *module, PyObject *path); static PyObject * os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2056,14 +2120,19 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *s; + PyObject *path; + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - s = args[0]; - return_value = os__path_isdir_impl(module, s); + path = args[0]; + _return_value = os__path_isdir_impl(module, path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: return return_value; @@ -2082,7 +2151,7 @@ PyDoc_STRVAR(os__path_isfile__doc__, #define OS__PATH_ISFILE_METHODDEF \ {"_path_isfile", _PyCFunction_CAST(os__path_isfile), METH_FASTCALL|METH_KEYWORDS, os__path_isfile__doc__}, -static PyObject * +static int os__path_isfile_impl(PyObject *module, PyObject *path); static PyObject * @@ -2116,13 +2185,18 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #undef KWTUPLE PyObject *argsbuf[1]; PyObject *path; + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } path = args[0]; - return_value = os__path_isfile_impl(module, path); + _return_value = os__path_isfile_impl(module, path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: return return_value; @@ -2132,20 +2206,20 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #if defined(MS_WINDOWS) -PyDoc_STRVAR(os__path_exists__doc__, -"_path_exists($module, /, path)\n" +PyDoc_STRVAR(os__path_islink__doc__, +"_path_islink($module, /, path)\n" "--\n" "\n" -"Test whether a path exists. Returns False for broken symbolic links"); +"Test whether a path is a symbolic link"); -#define OS__PATH_EXISTS_METHODDEF \ - {"_path_exists", _PyCFunction_CAST(os__path_exists), METH_FASTCALL|METH_KEYWORDS, os__path_exists__doc__}, +#define OS__PATH_ISLINK_METHODDEF \ + {"_path_islink", _PyCFunction_CAST(os__path_islink), METH_FASTCALL|METH_KEYWORDS, os__path_islink__doc__}, -static PyObject * -os__path_exists_impl(PyObject *module, PyObject *path); +static int +os__path_islink_impl(PyObject *module, PyObject *path); static PyObject * -os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2169,19 +2243,24 @@ os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj static const char * const _keywords[] = {"path", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_path_exists", + .fname = "_path_islink", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[1]; PyObject *path; + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } path = args[0]; - return_value = os__path_exists_impl(module, path); + _return_value = os__path_islink_impl(module, path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: return return_value; @@ -2191,20 +2270,20 @@ os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #if defined(MS_WINDOWS) -PyDoc_STRVAR(os__path_islink__doc__, -"_path_islink($module, /, path)\n" +PyDoc_STRVAR(os__path_isjunction__doc__, +"_path_isjunction($module, /, path)\n" "--\n" "\n" -"Test whether a path is a symbolic link"); +"Test whether a path is a junction"); -#define OS__PATH_ISLINK_METHODDEF \ - {"_path_islink", _PyCFunction_CAST(os__path_islink), METH_FASTCALL|METH_KEYWORDS, os__path_islink__doc__}, +#define OS__PATH_ISJUNCTION_METHODDEF \ + {"_path_isjunction", _PyCFunction_CAST(os__path_isjunction), METH_FASTCALL|METH_KEYWORDS, os__path_isjunction__doc__}, -static PyObject * -os__path_islink_impl(PyObject *module, PyObject *path); +static int +os__path_isjunction_impl(PyObject *module, PyObject *path); static PyObject * -os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2228,19 +2307,24 @@ os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj static const char * const _keywords[] = {"path", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_path_islink", + .fname = "_path_isjunction", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[1]; PyObject *path; + int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } path = args[0]; - return_value = os__path_islink_impl(module, path); + _return_value = os__path_isjunction_impl(module, path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: return return_value; @@ -12097,6 +12181,14 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS__PATH_SPLITROOT_METHODDEF #endif /* !defined(OS__PATH_SPLITROOT_METHODDEF) */ +#ifndef OS__PATH_EXISTS_METHODDEF + #define OS__PATH_EXISTS_METHODDEF +#endif /* !defined(OS__PATH_EXISTS_METHODDEF) */ + +#ifndef OS__PATH_LEXISTS_METHODDEF + #define OS__PATH_LEXISTS_METHODDEF +#endif /* !defined(OS__PATH_LEXISTS_METHODDEF) */ + #ifndef OS__PATH_ISDIR_METHODDEF #define OS__PATH_ISDIR_METHODDEF #endif /* !defined(OS__PATH_ISDIR_METHODDEF) */ @@ -12105,14 +12197,14 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS__PATH_ISFILE_METHODDEF #endif /* !defined(OS__PATH_ISFILE_METHODDEF) */ -#ifndef OS__PATH_EXISTS_METHODDEF - #define OS__PATH_EXISTS_METHODDEF -#endif /* !defined(OS__PATH_EXISTS_METHODDEF) */ - #ifndef OS__PATH_ISLINK_METHODDEF #define OS__PATH_ISLINK_METHODDEF #endif /* !defined(OS__PATH_ISLINK_METHODDEF) */ +#ifndef OS__PATH_ISJUNCTION_METHODDEF + #define OS__PATH_ISJUNCTION_METHODDEF +#endif /* !defined(OS__PATH_ISJUNCTION_METHODDEF) */ + #ifndef OS_NICE_METHODDEF #define OS_NICE_METHODDEF #endif /* !defined(OS_NICE_METHODDEF) */ @@ -12660,4 +12752,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=c4698b47007cd6eb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=af5074c4ce4b19f1 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 710a171580a438..07fec35cb32d90 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5088,384 +5088,363 @@ os__path_splitroot_impl(PyObject *module, path_t *path) } -/*[clinic input] -os._path_isdir - - s: 'O' +#define PY_IFREG 1 // Regular file +#define PY_IFDIR 2 // Directory +#define PY_IFLNK 4 // Symlink +#define PY_IFMNT 8 // Mount Point (junction) +#define PY_IFLRP 16 // Link Reparse Point (name-surrogate, symlink, junction) +#define PY_IFRRP 32 // Regular Reparse Point + +static inline BOOL +_testInfo(DWORD attributes, DWORD reparseTag, BOOL diskDevice, int testedType) +{ + switch (testedType) { + case PY_IFREG: + return diskDevice && attributes && + !(attributes & FILE_ATTRIBUTE_DIRECTORY); + case PY_IFDIR: + return attributes & FILE_ATTRIBUTE_DIRECTORY; + case PY_IFLNK: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag == IO_REPARSE_TAG_SYMLINK; + case PY_IFMNT: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag == IO_REPARSE_TAG_MOUNT_POINT; + case PY_IFLRP: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + IsReparseTagNameSurrogate(reparseTag); + case PY_IFRRP: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag && !IsReparseTagNameSurrogate(reparseTag); + } + + return FALSE; +} -Return true if the pathname refers to an existing directory. +static BOOL +_testFileTypeByHandle(HANDLE hfile, int testedType, BOOL diskOnly) +{ + assert(testedType == PY_IFREG || testedType == PY_IFDIR || + testedType == PY_IFLNK || testedType == PY_IFMNT || + testedType == PY_IFLRP || testedType == PY_IFRRP); -[clinic start generated code]*/ + BOOL diskDevice = GetFileType(hfile) == FILE_TYPE_DISK; + if (diskOnly && !diskDevice) { + return FALSE; + } + if (testedType != PY_IFREG && testedType != PY_IFDIR) { + FILE_ATTRIBUTE_TAG_INFO info; + return GetFileInformationByHandleEx(hfile, FileAttributeTagInfo, &info, + sizeof(info)) && + _testInfo(info.FileAttributes, info.ReparseTag, diskDevice, + testedType); + } + FILE_BASIC_INFO info; + return GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, + sizeof(info)) && + _testInfo(info.FileAttributes, 0, diskDevice, testedType); +} -static PyObject * -os__path_isdir_impl(PyObject *module, PyObject *s) -/*[clinic end generated code: output=9d87ab3c8b8a4e61 input=c17f7ef21d22d64e]*/ +static BOOL +_testFileTypeByName(LPCWSTR path, int testedType) { - HANDLE hfile; - BOOL close_file = TRUE; - FILE_BASIC_INFO info; - path_t _path = PATH_T_INITIALIZE("isdir", "s", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + assert(testedType == PY_IFREG || testedType == PY_IFDIR || + testedType == PY_IFLNK || testedType == PY_IFMNT || + testedType == PY_IFLRP || testedType == PY_IFRRP); - if (!path_converter(s, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; + FILE_STAT_BASIC_INFORMATION info; + if (_Py_GetFileInformationByName(path, FileStatBasicByNameInfo, &info, + sizeof(info))) + { + BOOL diskDevice = info.DeviceType == FILE_DEVICE_DISK || + info.DeviceType == FILE_DEVICE_VIRTUAL_DISK || + info.DeviceType == FILE_DEVICE_CD_ROM; + BOOL result = _testInfo(info.FileAttributes, info.ReparseTag, + diskDevice, testedType); + if (!result || (testedType != PY_IFREG && testedType != PY_IFDIR) || + !(info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) + { + return result; } - return NULL; + } + else if (_Py_GetFileInformationByName_ErrorIsTrustworthy( + GetLastError())) + { + return FALSE; } - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY; - } else if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY)) { - slow_path = FALSE; - result = 0; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } + DWORD flags = FILE_FLAG_BACKUP_SEMANTICS; + if (testedType != PY_IFREG && testedType != PY_IFDIR) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; + HANDLE hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, + OPEN_EXISTING, flags, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + BOOL result = _testFileTypeByHandle(hfile, testedType, FALSE); + CloseHandle(hfile); + return result; + } + + switch (GetLastError()) { + case ERROR_ACCESS_DENIED: + case ERROR_SHARING_VIOLATION: + case ERROR_CANT_ACCESS_FILE: + case ERROR_INVALID_PARAMETER: + int rc; + STRUCT_STAT st; + if (testedType == PY_IFREG || testedType == PY_IFDIR) { + rc = STAT(path, &st); } else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + // PY_IFRRP is not generally supported in this case, except for + // unhandled reparse points such as IO_REPARSE_TAG_APPEXECLINK. + rc = LSTAT(path, &st); } - if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, - sizeof(info))) - { - result = info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY; - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISDIR(st.st_mode); - } - break; - default: - result = 0; - } + if (!rc) { + return _testInfo(st.st_file_attributes, st.st_reparse_tag, + st.st_mode & S_IFREG, testedType); } } - Py_END_ALLOW_THREADS - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return FALSE; } -/*[clinic input] -os._path_isfile +static BOOL +_testFileExistsByName(LPCWSTR path, BOOL followLinks) +{ + FILE_STAT_BASIC_INFORMATION info; + if (_Py_GetFileInformationByName(path, FileStatBasicByNameInfo, &info, + sizeof(info))) + { + if (!(info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) || + !followLinks && IsReparseTagNameSurrogate(info.ReparseTag)) + { + return TRUE; + } + } + else if (_Py_GetFileInformationByName_ErrorIsTrustworthy( + GetLastError())) + { + return FALSE; + } - path: 'O' + DWORD flags = FILE_FLAG_BACKUP_SEMANTICS; + if (!followLinks) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + HANDLE hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, + OPEN_EXISTING, flags, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + if (followLinks) { + CloseHandle(hfile); + return TRUE; + } + // Regular Reparse Points (PY_IFRRP) have to be traversed. + BOOL result = _testFileTypeByHandle(hfile, PY_IFRRP, FALSE); + CloseHandle(hfile); + if (!result) { + return TRUE; + } + hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + CloseHandle(hfile); + return TRUE; + } + } -Test whether a path is a regular file + switch (GetLastError()) { + case ERROR_ACCESS_DENIED: + case ERROR_SHARING_VIOLATION: + case ERROR_CANT_ACCESS_FILE: + case ERROR_INVALID_PARAMETER: + STRUCT_STAT _st; + return followLinks ? !STAT(path, &_st): !LSTAT(path, &_st); + } -[clinic start generated code]*/ + return FALSE; +} -static PyObject * -os__path_isfile_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=2394ed7c4b5cfd85 input=de22d74960ade365]*/ -{ - HANDLE hfile; - BOOL close_file = TRUE; - FILE_BASIC_INFO info; - path_t _path = PATH_T_INITIALIZE("isfile", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; - if (!path_converter(path, &_path)) { - path_cleanup(&_path); +static int +_testFileExists(path_t *_path, PyObject *path, BOOL followLinks) +{ + BOOL result = FALSE; + if (!path_converter(path, _path)) { + path_cleanup(_path); if (PyErr_ExceptionMatches(PyExc_ValueError)) { PyErr_Clear(); - Py_RETURN_FALSE; + return FALSE; } - return NULL; + return -1; } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = !(statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY); - } else if (statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - slow_path = FALSE; - result = 0; + if (_path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(_path->fd); + if (hfile != INVALID_HANDLE_VALUE) { + if (GetFileType(hfile) != FILE_TYPE_UNKNOWN || !GetLastError()) { + result = TRUE; } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; } } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + else if (_path->wide) { + result = _testFileExistsByName(_path->wide, followLinks); + } + Py_END_ALLOW_THREADS + + path_cleanup(_path); + return result; +} + + +static int +_testFileType(path_t *_path, PyObject *path, int testedType) +{ + BOOL result = FALSE; + if (!path_converter(path, _path)) { + path_cleanup(_path); + if (PyErr_ExceptionMatches(PyExc_ValueError)) { + PyErr_Clear(); + return FALSE; } + return -1; + } + + Py_BEGIN_ALLOW_THREADS + if (_path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(_path->fd); if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, - sizeof(info))) - { - result = !(info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY); - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISREG(st.st_mode); - } - break; - default: - result = 0; - } + result = _testFileTypeByHandle(hfile, testedType, TRUE); } } + else if (_path->wide) { + result = _testFileTypeByName(_path->wide, testedType); + } Py_END_ALLOW_THREADS - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + path_cleanup(_path); + return result; } /*[clinic input] -os._path_exists +os._path_exists -> bool - path: 'O' + path: object + / -Test whether a path exists. Returns False for broken symbolic links +Test whether a path exists. Returns False for broken symbolic links. [clinic start generated code]*/ -static PyObject * +static int os__path_exists_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=f508c3b35e13a249 input=380f77cdfa0f7ae8]*/ +/*[clinic end generated code: output=8f784b3abf9f8588 input=2777da15bc4ba5a3]*/ { - HANDLE hfile; - BOOL close_file = TRUE; - path_t _path = PATH_T_INITIALIZE("exists", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + path_t _path = PATH_T_INITIALIZE("_path_exists", "path", 0, 1); + return _testFileExists(&_path, path, TRUE); +} - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; - } - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = 1; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } - } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - } - if (hfile != INVALID_HANDLE_VALUE) { - result = 1; - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = 1; - } - break; - default: - result = 0; - } - } - } - Py_END_ALLOW_THREADS +/*[clinic input] +os._path_lexists -> bool - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + path: object + / + +Test whether a path exists. Returns True for broken symbolic links. + +[clinic start generated code]*/ + +static int +os__path_lexists_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=fec4a91cf4ffccf1 input=8843d4d6d4e7c779]*/ +{ + path_t _path = PATH_T_INITIALIZE("_path_lexists", "path", 0, 1); + return _testFileExists(&_path, path, FALSE); +} + + +/*[clinic input] +os._path_isdir -> bool + + s as path: object + +Return true if the pathname refers to an existing directory. + +[clinic start generated code]*/ + +static int +os__path_isdir_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=0504fd403f369701 input=2cb54dd97eb970f7]*/ +{ + path_t _path = PATH_T_INITIALIZE("_path_isdir", "s", 0, 1); + return _testFileType(&_path, path, PY_IFDIR); +} + + +/*[clinic input] +os._path_isfile -> bool + + path: object + +Test whether a path is a regular file + +[clinic start generated code]*/ + +static int +os__path_isfile_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=b40d620efe5a896f input=54b428a310debaea]*/ +{ + path_t _path = PATH_T_INITIALIZE("_path_isfile", "path", 0, 1); + return _testFileType(&_path, path, PY_IFREG); } /*[clinic input] -os._path_islink +os._path_islink -> bool - path: 'O' + path: object Test whether a path is a symbolic link [clinic start generated code]*/ -static PyObject * +static int os__path_islink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=6d8640b1a390c054 input=38a3cb937ccf59bf]*/ +/*[clinic end generated code: output=9d0cf8e4c640dfe6 input=b71fed60b9b2cd73]*/ { - HANDLE hfile; - BOOL close_file = TRUE; - FILE_ATTRIBUTE_TAG_INFO info; - path_t _path = PATH_T_INITIALIZE("islink", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + path_t _path = PATH_T_INITIALIZE("_path_islink", "path", 0, 1); + return _testFileType(&_path, path, PY_IFLNK); +} - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; - } - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - slow_path = FALSE; - if (statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - result = (statInfo.ReparseTag == IO_REPARSE_TAG_SYMLINK); - } - else { - result = 0; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } - } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, - FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, - NULL); - } - if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileAttributeTagInfo, &info, - sizeof(info))) - { - result = (info.ReparseTag == IO_REPARSE_TAG_SYMLINK); - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (LSTAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISLNK(st.st_mode); - } - break; - default: - result = 0; - } - } - } - Py_END_ALLOW_THREADS +/*[clinic input] +os._path_isjunction -> bool - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + path: object + +Test whether a path is a junction + +[clinic start generated code]*/ + +static int +os__path_isjunction_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=f1d51682a077654d input=103ccedcdb714f11]*/ +{ + path_t _path = PATH_T_INITIALIZE("_path_isjunction", "path", 0, 1); + return _testFileType(&_path, path, PY_IFMNT); } +#undef PY_IFREG +#undef PY_IFDIR +#undef PY_IFLNK +#undef PY_IFMNT +#undef PY_IFLRP +#undef PY_IFRRP + #endif /* MS_WINDOWS */ @@ -16921,7 +16900,9 @@ static PyMethodDef posix_methods[] = { OS__PATH_ISDIR_METHODDEF OS__PATH_ISFILE_METHODDEF OS__PATH_ISLINK_METHODDEF + OS__PATH_ISJUNCTION_METHODDEF OS__PATH_EXISTS_METHODDEF + OS__PATH_LEXISTS_METHODDEF OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF {NULL, NULL} /* Sentinel */ From 6e9863d7a3516cc76d6ce13923b15620499f3855 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 21 May 2024 16:42:51 -0400 Subject: [PATCH 155/903] gh-118692: Avoid creating unnecessary StopIteration instances for monitoring (#119216) --- Doc/c-api/monitoring.rst | 6 ++-- Include/cpython/monitoring.h | 6 ++-- Include/internal/pycore_opcode_metadata.h | 4 +-- Lib/test/test_monitoring.py | 31 ++++++++++++++++--- ...-05-20-14-57-39.gh-issue-118692.Qadm7F.rst | 1 + Modules/_testcapi/monitoring.c | 9 +++--- Python/bytecodes.c | 8 ++--- Python/ceval.c | 14 +++++++-- Python/generated_cases.c.h | 8 ++--- Python/instrumentation.c | 8 +++-- 10 files changed, 60 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-20-14-57-39.gh-issue-118692.Qadm7F.rst diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 763ec8ef761e4e..ec743b98ba7024 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -121,10 +121,10 @@ See :mod:`sys.monitoring` for descriptions of the events. :c:func:`PyErr_GetRaisedException`). -.. c:function:: int PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +.. c:function:: int PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) - Fire a ``STOP_ITERATION`` event with the current exception (as returned by - :c:func:`PyErr_GetRaisedException`). + Fire a ``STOP_ITERATION`` event. If ``value`` is an instance of :exc:`StopIteration`, it is used. Otherwise, + a new :exc:`StopIteration` instance is created with ``value`` as its argument. Managing the Monitoring State diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index efb9ec0e587552..797ba51246b1c6 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -101,7 +101,7 @@ PyAPI_FUNC(int) _PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); PyAPI_FUNC(int) -_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value); #define _PYMONITORING_IF_ACTIVE(STATE, X) \ @@ -240,11 +240,11 @@ PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int } static inline int -PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) { _PYMONITORING_IF_ACTIVE( state, - _PyMonitoring_FireStopIterationEvent(state, codelike, offset)); + _PyMonitoring_FireStopIterationEvent(state, codelike, offset, value)); } #undef _PYMONITORING_IF_ACTIVE diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 2a237bc6dd8ee5..665b8627dfcc52 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1060,8 +1060,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 6974bc5517ae5f..b7c6abed1016dc 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1938,19 +1938,28 @@ def setUp(self): ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)), ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)), ( 1, E.PY_UNWIND, capi.fire_event_py_unwind, ValueError(6)), - ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, ValueError(7)), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, 7), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, StopIteration(8)), ] + self.EXPECT_RAISED_EXCEPTION = [E.PY_THROW, E.RAISE, E.EXCEPTION_HANDLED, E.PY_UNWIND] - def check_event_count(self, event, func, args, expected): + + def check_event_count(self, event, func, args, expected, callback_raises=None): class Counter: - def __init__(self): + def __init__(self, callback_raises): + self.callback_raises = callback_raises self.count = 0 + def __call__(self, *args): self.count += 1 + if self.callback_raises: + exc = self.callback_raises + self.callback_raises = None + raise exc try: - counter = Counter() + counter = Counter(callback_raises) sys.monitoring.register_callback(TEST_TOOL, event, counter) if event == E.C_RETURN or event == E.C_RAISE: sys.monitoring.set_events(TEST_TOOL, E.CALL) @@ -1987,8 +1996,9 @@ def test_fire_event(self): def test_missing_exception(self): for _, event, function, *args in self.cases: - if not (args and isinstance(args[-1], BaseException)): + if event not in self.EXPECT_RAISED_EXCEPTION: continue + assert args and isinstance(args[-1], BaseException) offset = 0 self.codelike = _testcapi.CodeLike(1) with self.subTest(function.__name__): @@ -1997,6 +2007,17 @@ def test_missing_exception(self): expected = ValueError(f"Firing event {evt} with no exception set") self.check_event_count(event, function, args_, expected) + def test_fire_event_failing_callback(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + exc = OSError(42) + with self.assertRaises(type(exc)): + self.check_event_count(event, function, args_, expected, + callback_raises=exc) + CANNOT_DISABLE = { E.PY_THROW, E.RAISE, E.RERAISE, E.EXCEPTION_HANDLED, E.PY_UNWIND } diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-20-14-57-39.gh-issue-118692.Qadm7F.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-20-14-57-39.gh-issue-118692.Qadm7F.rst new file mode 100644 index 00000000000000..11d177886df5b5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-20-14-57-39.gh-issue-118692.Qadm7F.rst @@ -0,0 +1 @@ +Avoid creating unnecessary :exc:`StopIteration` instances for monitoring. diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c index aa90cfc06c1536..6fd4a405688f48 100644 --- a/Modules/_testcapi/monitoring.c +++ b/Modules/_testcapi/monitoring.c @@ -416,16 +416,17 @@ fire_event_stop_iteration(PyObject *self, PyObject *args) { PyObject *codelike; int offset; - PyObject *exception; - if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + PyObject *value; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &value)) { return NULL; } - NULLABLE(exception); + NULLABLE(value); + PyObject *exception = NULL; PyMonitoringState *state = setup_fire(codelike, offset, exception); if (state == NULL) { return NULL; } - int res = PyMonitoring_FireStopIterationEvent(state, codelike, offset); + int res = PyMonitoring_FireStopIterationEvent(state, codelike, offset, value); RETURN_INT(teardown_fire(res, state, exception)); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 55eda9711dea1f..434eb804213810 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -292,11 +292,9 @@ dummy_func( /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { + if (monitor_stop_iteration(tstate, frame, this_instr, value)) { ERROR_NO_POP(); } - PyErr_SetRaisedException(NULL); } DECREF_INPUTS(); } @@ -307,11 +305,9 @@ dummy_func( tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { + if (monitor_stop_iteration(tstate, frame, this_instr, value)) { ERROR_NO_POP(); } - PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); } diff --git a/Python/ceval.c b/Python/ceval.c index 128e0417a9fd63..324d062fe9bb43 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -231,7 +231,8 @@ static void monitor_reraise(PyThreadState *tstate, _Py_CODEUNIT *instr); static int monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr); + _Py_CODEUNIT *instr, + PyObject *value); static void monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); @@ -2215,12 +2216,19 @@ monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, static int monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) + _Py_CODEUNIT *instr, PyObject *value) { if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { return 0; } - return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); + assert(!PyErr_Occurred()); + PyErr_SetObject(PyExc_StopIteration, value); + int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); + if (res < 0) { + return res; + } + PyErr_SetRaisedException(NULL); + return 0; } static void diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8b8112209cc78a..96161c5a6586fd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3260,11 +3260,9 @@ /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { + if (monitor_stop_iteration(tstate, frame, this_instr, value)) { goto error; } - PyErr_SetRaisedException(NULL); } Py_DECREF(value); stack_pointer += -1; @@ -3281,11 +3279,9 @@ value = stack_pointer[-1]; receiver = stack_pointer[-2]; if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { + if (monitor_stop_iteration(tstate, frame, this_instr, value)) { goto error; } - PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); stack_pointer[-2] = value; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 77d3489afcfc72..3d78214738e66b 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -2622,7 +2622,7 @@ exception_event_teardown(int err, PyObject *exc) { } else { assert(PyErr_Occurred()); - Py_DECREF(exc); + Py_XDECREF(exc); } return err; } @@ -2712,15 +2712,17 @@ _PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, in } int -_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) { int event = PY_MONITORING_EVENT_STOP_ITERATION; assert(state->active); + assert(!PyErr_Occurred()); + PyErr_SetObject(PyExc_StopIteration, value); PyObject *exc; if (exception_event_setup(&exc, event) < 0) { return -1; } PyObject *args[4] = { NULL, NULL, NULL, exc }; int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); - return exception_event_teardown(err, exc); + return exception_event_teardown(err, NULL); } From 9fa206aaeccc979a4bd03852ba38c045294a3d6f Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Tue, 21 May 2024 17:49:23 -0400 Subject: [PATCH 156/903] Docs: Add central references to free-threading-related options (#119017) --- Doc/c-api/module.rst | 4 ++-- Doc/using/cmdline.rst | 5 +++-- Doc/using/configure.rst | 2 +- Doc/whatsnew/3.13.rst | 3 ++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 6fe1ce9e994832..63e3bed6727987 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -427,14 +427,14 @@ The available slot types are: This slot is ignored by Python builds not configured with :option:`--disable-gil`. Otherwise, it determines whether or not importing this module will cause the GIL to be automatically enabled. See - :envvar:`PYTHON_GIL` and :option:`-X gil <-X>` for more detail. + :ref:`free-threaded-cpython` for more detail. Multiple ``Py_mod_gil`` slots may not be specified in one module definition. If ``Py_mod_gil`` is not specified, the import machinery defaults to ``Py_MOD_GIL_USED``. - .. versionadded: 3.13 + .. versionadded:: 3.13 See :PEP:`489` for more details on multi-phase initialization. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index dcca9060899a0f..7b2019625eb27a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -622,7 +622,8 @@ Miscellaneous options * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, respectively. Only available in builds configured with - :option:`--disable-gil`. See also :envvar:`PYTHON_GIL`. + :option:`--disable-gil`. See also :envvar:`PYTHON_GIL` and + :ref:`free-threaded-cpython`. .. versionadded:: 3.13 @@ -1212,7 +1213,7 @@ conflict. forced on. Setting it to ``0`` forces the GIL off. See also the :option:`-X gil <-X>` command-line option, which takes - precedence over this variable. + precedence over this variable, and :ref:`free-threaded-cpython`. Needs Python configured with the :option:`--disable-gil` build option. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index d30356d2058eff..428ee5275276a0 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -299,7 +299,7 @@ General Options Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to :data:`sys.abiflags`. - See :pep:`703` "Making the Global Interpreter Lock Optional in CPython". + See :ref:`free-threaded-cpython` for more detail. .. versionadded:: 3.13 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 8cd174f01ff600..7edfdd4f8167a0 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -368,7 +368,8 @@ CPython will run with the :term:`global interpreter lock` (GIL) disabled when configured using the ``--disable-gil`` option at build time. This is an experimental feature and therefore isn't used by default. Users need to either compile their own interpreter, or install one of the experimental -builds that are marked as *free-threaded*. +builds that are marked as *free-threaded*. See :pep:`703` "Making the Global +Interpreter Lock Optional in CPython" for more detail. Free-threaded execution allows for full utilization of the available processing power by running threads in parallel on available CPU cores. From a3e4fec8734a304d654e4ae24a4aa2f41a7b0640 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 21 May 2024 19:16:56 -0400 Subject: [PATCH 157/903] gh-118893: Evaluate all statements in the new REPL separately (#119318) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_pyrepl/simple_interact.py | 33 ++++++++-- Lib/code.py | 5 +- Lib/test/test_pyrepl/test_interact.py | 92 +++++++++++++++++++++++++++ Lib/test/test_traceback.py | 2 +- Lib/traceback.py | 5 +- 5 files changed, 128 insertions(+), 9 deletions(-) create mode 100644 Lib/test/test_pyrepl/test_interact.py diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 31b2097a78a226..d65b6d0d62790a 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -30,6 +30,7 @@ import linecache import sys import code +import ast from types import ModuleType from .readline import _get_reader, multiline_input @@ -74,9 +75,36 @@ def __init__( super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] self.can_colorize = _colorize.can_colorize() + def showsyntaxerror(self, filename=None): + super().showsyntaxerror(colorize=self.can_colorize) + def showtraceback(self): super().showtraceback(colorize=self.can_colorize) + def runsource(self, source, filename="", symbol="single"): + try: + tree = ast.parse(source) + except (OverflowError, SyntaxError, ValueError): + self.showsyntaxerror(filename) + return False + if tree.body: + *_, last_stmt = tree.body + for stmt in tree.body: + wrapper = ast.Interactive if stmt is last_stmt else ast.Module + the_symbol = symbol if stmt is last_stmt else "exec" + item = wrapper([stmt]) + try: + code = compile(item, filename, the_symbol) + except (OverflowError, ValueError): + self.showsyntaxerror(filename) + return False + + if code is None: + return True + + self.runcode(code) + return False + def run_multiline_interactive_console( mainmodule: ModuleType | None= None, future_flags: int = 0 @@ -144,10 +172,7 @@ def more_lines(unicodetext: str) -> bool: input_name = f"" linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] - symbol = "single" if not contains_pasted_code else "exec" - more = console.push(_strip_final_indent(statement), filename=input_name, _symbol=symbol) # type: ignore[call-arg] - if contains_pasted_code and more: - more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] + more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] assert not more input_n += 1 except KeyboardInterrupt: diff --git a/Lib/code.py b/Lib/code.py index 0c2fd2963b2118..b93902ccf545b3 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -94,7 +94,7 @@ def runcode(self, code): except: self.showtraceback() - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Display the syntax error that just occurred. This doesn't display a stack trace because there isn't one. @@ -106,6 +106,7 @@ def showsyntaxerror(self, filename=None): The output is written by self.write(), below. """ + colorize = kwargs.pop('colorize', False) type, value, tb = sys.exc_info() sys.last_exc = value sys.last_type = type @@ -123,7 +124,7 @@ def showsyntaxerror(self, filename=None): value = SyntaxError(msg, (filename, lineno, offset, line)) sys.last_exc = sys.last_value = value if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception_only(type, value) + lines = traceback.format_exception_only(type, value, colorize=colorize) self.write(''.join(lines)) else: # If someone has set sys.excepthook, we let that take precedence diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py new file mode 100644 index 00000000000000..b3dc07c063f90e --- /dev/null +++ b/Lib/test/test_pyrepl/test_interact.py @@ -0,0 +1,92 @@ +import contextlib +import io +import unittest +from unittest.mock import patch +from textwrap import dedent + +from test.support import force_not_colorized + +from _pyrepl.simple_interact import InteractiveColoredConsole + + +class TestSimpleInteract(unittest.TestCase): + def test_multiple_statements(self): + namespace = {} + code = dedent("""\ + class A: + def foo(self): + + + pass + + class B: + def bar(self): + pass + + a = 1 + a + """) + console = InteractiveColoredConsole(namespace, filename="") + with ( + patch.object(InteractiveColoredConsole, "showsyntaxerror") as showsyntaxerror, + patch.object(InteractiveColoredConsole, "runsource", wraps=console.runsource) as runsource, + ): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + showsyntaxerror.assert_not_called() + + + def test_multiple_statements_output(self): + namespace = {} + code = dedent("""\ + b = 1 + b + a = 1 + a + """) + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with contextlib.redirect_stdout(f): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + self.assertEqual(f.getvalue(), "1\n") + + def test_empty(self): + namespace = {} + code = "" + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with contextlib.redirect_stdout(f): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + self.assertEqual(f.getvalue(), "") + + def test_runsource_compiles_and_runs_code(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!')" + with patch.object(console, "runcode") as mock_runcode: + console.runsource(source) + mock_runcode.assert_called_once() + + def test_runsource_returns_false_for_successful_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!')" + result = console.runsource(source) + self.assertFalse(result) + + @force_not_colorized + def test_runsource_returns_false_for_failed_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!'" + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertIn('SyntaxError', f.getvalue()) + + def test_runsource_shows_syntax_error_for_failed_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!'" + with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: + console.runsource(source) + mock_showsyntaxerror.assert_called_once() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 5987ec382e6c85..5035de114b5e9d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -543,7 +543,7 @@ def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.format_exception_only)), - '(exc, /, value=, *, show_group=False)') + '(exc, /, value=, *, show_group=False, **kwargs)') class PurePythonExceptionFormattingMixin: diff --git a/Lib/traceback.py b/Lib/traceback.py index 9401b461497cc1..280d92d04cac9b 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -155,7 +155,7 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ return list(te.format(chain=chain, colorize=colorize)) -def format_exception_only(exc, /, value=_sentinel, *, show_group=False): +def format_exception_only(exc, /, value=_sentinel, *, show_group=False, **kwargs): """Format the exception part of a traceback. The return value is a list of strings, each ending in a newline. @@ -170,10 +170,11 @@ def format_exception_only(exc, /, value=_sentinel, *, show_group=False): :exc:`BaseExceptionGroup`, the nested exceptions are included as well, recursively, with indentation relative to their nesting depth. """ + colorize = kwargs.get("colorize", False) if value is _sentinel: value = exc te = TracebackException(type(value), value, None, compact=True) - return list(te.format_exception_only(show_group=show_group)) + return list(te.format_exception_only(show_group=show_group, colorize=colorize)) # -- not official API but folk probably use these two functions. From 506b1a3ff66a41c72d205c8e4cba574e439d8e76 Mon Sep 17 00:00:00 2001 From: Koudai Aono Date: Tue, 21 May 2024 19:22:21 -0400 Subject: [PATCH 158/903] gh-119205: Fix autocompletion bug in new repl (#119229) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_pyrepl/readline.py | 4 ++- Lib/test/test_pyrepl/test_pyrepl.py | 30 ++++++++++++++++---- Lib/test/test_pyrepl/test_unix_eventqueue.py | 2 +- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 9c85ce175fd6d6..787dbc06e58e18 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -28,6 +28,7 @@ from __future__ import annotations +import warnings from dataclasses import dataclass, field import os @@ -301,7 +302,8 @@ def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> reader.more_lines = more_lines reader.ps1 = reader.ps2 = ps1 reader.ps3 = reader.ps4 = ps2 - return reader.readline(), reader.was_paste_mode_activated + with warnings.catch_warnings(action="ignore"): + return reader.readline(), reader.was_paste_mode_activated finally: reader.more_lines = saved reader.paste_mode = False diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index bc0a9975e34e00..b643ae5895c97e 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1,12 +1,15 @@ import itertools +import io import os import rlcompleter -import unittest from unittest import TestCase +from unittest.mock import patch -from .support import FakeConsole, handle_all_events, handle_events_narrow_console, multiline_input, code_to_events +from .support import FakeConsole, handle_all_events, handle_events_narrow_console +from .support import more_lines, multiline_input, code_to_events from _pyrepl.console import Event from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig +from _pyrepl.readline import multiline_input as readline_multiline_input class TestCursorPosition(TestCase): @@ -475,6 +478,25 @@ def test_updown_arrow_with_completion_menu(self): output = multiline_input(reader, namespace) self.assertEqual(output, "os.") + @patch("_pyrepl.readline._ReadlineWrapper.get_reader") + @patch("sys.stderr", new_callable=io.StringIO) + def test_completion_with_warnings(self, mock_stderr, mock_get_reader): + class Dummy: + @property + def test_func(self): + import warnings + warnings.warn("warnings\n") + return None + + dummy = Dummy() + events = code_to_events("dummy.test_func.\t\n\n") + namespace = {"dummy": dummy} + reader = self.prepare_reader(events, namespace) + mock_get_reader.return_value = reader + output = readline_multiline_input(more_lines, ">>>", "...") + self.assertEqual(output[0], "dummy.test_func.__") + self.assertEqual(mock_stderr.getvalue(), "") + class TestPasteEvent(TestCase): def prepare_reader(self, events): @@ -633,7 +655,3 @@ def test_bracketed_paste_single_line(self): reader = self.prepare_reader(events) output = multiline_input(reader) self.assertEqual(output, input_code) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_pyrepl/test_unix_eventqueue.py b/Lib/test/test_pyrepl/test_unix_eventqueue.py index be2bc00e5692a7..c06536b4a86a04 100644 --- a/Lib/test/test_pyrepl/test_unix_eventqueue.py +++ b/Lib/test/test_pyrepl/test_unix_eventqueue.py @@ -7,7 +7,7 @@ @patch("_pyrepl.curses.tigetstr", lambda x: b"") -class TestUnivEventQueue(unittest.TestCase): +class TestUnixEventQueue(unittest.TestCase): def setUp(self): self.file = tempfile.TemporaryFile() From 98e855fcc1f1d490c803565e84cb611b3f057e45 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 21 May 2024 20:46:39 -0400 Subject: [PATCH 159/903] gh-119180: Add LOAD_COMMON_CONSTANT opcode (#119321) The PEP 649 implementation will require a way to load NotImplementedError from the bytecode. @markshannon suggested implementing this by converting LOAD_ASSERTION_ERROR into a more general mechanism for loading constants. This PR adds this new opcode. I will work on the rest of the implementation of the PEP separately. Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Doc/library/dis.rst | 9 +- Include/internal/pycore_opcode_metadata.h | 16 +- Include/internal/pycore_opcode_utils.h | 5 + Include/internal/pycore_uop_ids.h | 2 +- Include/internal/pycore_uop_metadata.h | 6 +- Include/opcode_ids.h | 122 ++++----- Lib/_opcode_metadata.py | 122 ++++----- Lib/dis.py | 8 + Lib/importlib/_bootstrap_external.py | 5 +- Lib/opcode.py | 1 + Lib/test/test_code.py | 2 +- Lib/test/test_compile.py | 2 +- Lib/test/test_dis.py | 234 +++++++++--------- ...-05-21-12-17-02.gh-issue-119180.UNDUb9.rst | 2 + Objects/codeobject.c | 2 +- Programs/test_frozenmain.h | 20 +- Python/bytecodes.c | 14 +- Python/compile.c | 2 +- Python/executor_cases.c.h | 15 +- Python/generated_cases.c.h | 32 ++- Python/opcode_targets.h | 2 +- Python/optimizer_cases.c.h | 2 +- 22 files changed, 337 insertions(+), 288 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-21-12-17-02.gh-issue-119180.UNDUb9.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index b9e2efab827384..fda46d260bcb46 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -929,12 +929,13 @@ iterations of the loop. Exception representation on the stack now consist of one, not three, items. -.. opcode:: LOAD_ASSERTION_ERROR +.. opcode:: LOAD_COMMON_CONSTANT - Pushes :exc:`AssertionError` onto the stack. Used by the :keyword:`assert` - statement. + Pushes a common constant onto the stack. The interpreter contains a hardcoded + list of constants supported by this instruction. Used by the :keyword:`assert` + statement to load :exc:`AssertionError`. - .. versionadded:: 3.9 + .. versionadded:: 3.14 .. opcode:: LOAD_BUILD_CLASS diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 665b8627dfcc52..6da4702b2cb2ee 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -273,8 +273,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + (oparg-1); case LIST_EXTEND: return 2 + (oparg-1); - case LOAD_ASSERTION_ERROR: - return 0; case LOAD_ATTR: return 1; case LOAD_ATTR_CLASS: @@ -303,6 +301,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case LOAD_BUILD_CLASS: return 0; + case LOAD_COMMON_CONSTANT: + return 0; case LOAD_CONST: return 0; case LOAD_DEREF: @@ -706,8 +706,6 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1 + (oparg-1); case LIST_EXTEND: return 1 + (oparg-1); - case LOAD_ASSERTION_ERROR: - return 1; case LOAD_ATTR: return 1 + (oparg & 1); case LOAD_ATTR_CLASS: @@ -736,6 +734,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1 + (oparg & 1); case LOAD_BUILD_CLASS: return 1; + case LOAD_COMMON_CONSTANT: + return 1; case LOAD_CONST: return 1; case LOAD_DEREF: @@ -1082,7 +1082,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, 0 }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, @@ -1097,6 +1096,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, @@ -1279,7 +1279,6 @@ _PyOpcode_macro_expansion[256] = { [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, 0, 0 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, 0, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, 0, 0 } } }, - [LOAD_ASSERTION_ERROR] = { .nuops = 1, .uops = { { _LOAD_ASSERTION_ERROR, 0, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, @@ -1292,6 +1291,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, 0, 0 } } }, + [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, 0, 0 } } }, [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, 0, 0 } } }, [LOAD_DEREF] = { .nuops = 1, .uops = { { _LOAD_DEREF, 0, 0 } } }, [LOAD_FAST] = { .nuops = 1, .uops = { { _LOAD_FAST, 0, 0 } } }, @@ -1488,7 +1488,6 @@ const char *_PyOpcode_OpName[268] = { [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", [LIST_APPEND] = "LIST_APPEND", [LIST_EXTEND] = "LIST_EXTEND", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [LOAD_ATTR] = "LOAD_ATTR", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", @@ -1504,6 +1503,7 @@ const char *_PyOpcode_OpName[268] = { [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [LOAD_CLOSURE] = "LOAD_CLOSURE", + [LOAD_COMMON_CONSTANT] = "LOAD_COMMON_CONSTANT", [LOAD_CONST] = "LOAD_CONST", [LOAD_DEREF] = "LOAD_DEREF", [LOAD_FAST] = "LOAD_FAST", @@ -1741,7 +1741,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [JUMP_FORWARD] = JUMP_FORWARD, [LIST_APPEND] = LIST_APPEND, [LIST_EXTEND] = LIST_EXTEND, - [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, [LOAD_ATTR] = LOAD_ATTR, [LOAD_ATTR_CLASS] = LOAD_ATTR, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, @@ -1756,6 +1755,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_ATTR_SLOT] = LOAD_ATTR, [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = LOAD_COMMON_CONSTANT, [LOAD_CONST] = LOAD_CONST, [LOAD_DEREF] = LOAD_DEREF, [LOAD_FAST] = LOAD_FAST, diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 208bfb2f75308b..b06e469dd5bd91 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -58,6 +58,11 @@ extern "C" { #define MAKE_FUNCTION_ANNOTATIONS 0x04 #define MAKE_FUNCTION_CLOSURE 0x08 +/* Values used as the oparg for LOAD_COMMON_CONSTANT */ +#define CONSTANT_ASSERTIONERROR 0 +#define CONSTANT_NOTIMPLEMENTEDERROR 1 +#define NUM_COMMON_CONSTANTS 2 + /* Values used in the oparg for RESUME */ #define RESUME_AT_FUNC_START 0 #define RESUME_AFTER_YIELD 1 diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 1e6ef8e54a221a..d36b172f57ec68 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -169,7 +169,6 @@ extern "C" { #define _JUMP_TO_TOP 392 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR #define _LOAD_ATTR 393 #define _LOAD_ATTR_CLASS 394 #define _LOAD_ATTR_CLASS_0 395 @@ -190,6 +189,7 @@ extern "C" { #define _LOAD_ATTR_SLOT_1 408 #define _LOAD_ATTR_WITH_HINT 409 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS +#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_INLINE 410 #define _LOAD_CONST_INLINE_BORROW 411 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 470e95e2b3b041..f555824fc8f3b3 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -93,7 +93,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, - [_LOAD_ASSERTION_ERROR] = 0, + [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, @@ -400,7 +400,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_JUMP_TO_TOP] = "_JUMP_TO_TOP", [_LIST_APPEND] = "_LIST_APPEND", [_LIST_EXTEND] = "_LIST_EXTEND", - [_LOAD_ASSERTION_ERROR] = "_LOAD_ASSERTION_ERROR", [_LOAD_ATTR] = "_LOAD_ATTR", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", [_LOAD_ATTR_CLASS_0] = "_LOAD_ATTR_CLASS_0", @@ -419,6 +418,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_SLOT_1] = "_LOAD_ATTR_SLOT_1", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", + [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", [_LOAD_CONST] = "_LOAD_CONST", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", @@ -662,7 +662,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _POP_EXCEPT: return 1; - case _LOAD_ASSERTION_ERROR: + case _LOAD_COMMON_CONSTANT: return 0; case _LOAD_BUILD_CLASS: return 0; diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 647f7c0ecb1ec8..6a608651d1e81d 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -33,66 +33,66 @@ extern "C" { #define GET_LEN 20 #define GET_YIELD_FROM_ITER 21 #define INTERPRETER_EXIT 22 -#define LOAD_ASSERTION_ERROR 23 -#define LOAD_BUILD_CLASS 24 -#define LOAD_LOCALS 25 -#define MAKE_FUNCTION 26 -#define MATCH_KEYS 27 -#define MATCH_MAPPING 28 -#define MATCH_SEQUENCE 29 -#define NOP 30 -#define POP_EXCEPT 31 -#define POP_TOP 32 -#define PUSH_EXC_INFO 33 -#define PUSH_NULL 34 -#define RETURN_GENERATOR 35 -#define RETURN_VALUE 36 -#define SETUP_ANNOTATIONS 37 -#define STORE_SLICE 38 -#define STORE_SUBSCR 39 -#define TO_BOOL 40 -#define UNARY_INVERT 41 -#define UNARY_NEGATIVE 42 -#define UNARY_NOT 43 -#define WITH_EXCEPT_START 44 -#define BINARY_OP 45 -#define BUILD_CONST_KEY_MAP 46 -#define BUILD_LIST 47 -#define BUILD_MAP 48 -#define BUILD_SET 49 -#define BUILD_SLICE 50 -#define BUILD_STRING 51 -#define BUILD_TUPLE 52 -#define CALL 53 -#define CALL_FUNCTION_EX 54 -#define CALL_INTRINSIC_1 55 -#define CALL_INTRINSIC_2 56 -#define CALL_KW 57 -#define COMPARE_OP 58 -#define CONTAINS_OP 59 -#define CONVERT_VALUE 60 -#define COPY 61 -#define COPY_FREE_VARS 62 -#define DELETE_ATTR 63 -#define DELETE_DEREF 64 -#define DELETE_FAST 65 -#define DELETE_GLOBAL 66 -#define DELETE_NAME 67 -#define DICT_MERGE 68 -#define DICT_UPDATE 69 -#define ENTER_EXECUTOR 70 -#define EXTENDED_ARG 71 -#define FOR_ITER 72 -#define GET_AWAITABLE 73 -#define IMPORT_FROM 74 -#define IMPORT_NAME 75 -#define IS_OP 76 -#define JUMP_BACKWARD 77 -#define JUMP_BACKWARD_NO_INTERRUPT 78 -#define JUMP_FORWARD 79 -#define LIST_APPEND 80 -#define LIST_EXTEND 81 -#define LOAD_ATTR 82 +#define LOAD_BUILD_CLASS 23 +#define LOAD_LOCALS 24 +#define MAKE_FUNCTION 25 +#define MATCH_KEYS 26 +#define MATCH_MAPPING 27 +#define MATCH_SEQUENCE 28 +#define NOP 29 +#define POP_EXCEPT 30 +#define POP_TOP 31 +#define PUSH_EXC_INFO 32 +#define PUSH_NULL 33 +#define RETURN_GENERATOR 34 +#define RETURN_VALUE 35 +#define SETUP_ANNOTATIONS 36 +#define STORE_SLICE 37 +#define STORE_SUBSCR 38 +#define TO_BOOL 39 +#define UNARY_INVERT 40 +#define UNARY_NEGATIVE 41 +#define UNARY_NOT 42 +#define WITH_EXCEPT_START 43 +#define BINARY_OP 44 +#define BUILD_CONST_KEY_MAP 45 +#define BUILD_LIST 46 +#define BUILD_MAP 47 +#define BUILD_SET 48 +#define BUILD_SLICE 49 +#define BUILD_STRING 50 +#define BUILD_TUPLE 51 +#define CALL 52 +#define CALL_FUNCTION_EX 53 +#define CALL_INTRINSIC_1 54 +#define CALL_INTRINSIC_2 55 +#define CALL_KW 56 +#define COMPARE_OP 57 +#define CONTAINS_OP 58 +#define CONVERT_VALUE 59 +#define COPY 60 +#define COPY_FREE_VARS 61 +#define DELETE_ATTR 62 +#define DELETE_DEREF 63 +#define DELETE_FAST 64 +#define DELETE_GLOBAL 65 +#define DELETE_NAME 66 +#define DICT_MERGE 67 +#define DICT_UPDATE 68 +#define ENTER_EXECUTOR 69 +#define EXTENDED_ARG 70 +#define FOR_ITER 71 +#define GET_AWAITABLE 72 +#define IMPORT_FROM 73 +#define IMPORT_NAME 74 +#define IS_OP 75 +#define JUMP_BACKWARD 76 +#define JUMP_BACKWARD_NO_INTERRUPT 77 +#define JUMP_FORWARD 78 +#define LIST_APPEND 79 +#define LIST_EXTEND 80 +#define LOAD_ATTR 81 +#define LOAD_COMMON_CONSTANT 82 #define LOAD_CONST 83 #define LOAD_DEREF 84 #define LOAD_FAST 85 @@ -235,7 +235,7 @@ extern "C" { #define SETUP_WITH 266 #define STORE_FAST_MAYBE_NULL 267 -#define HAVE_ARGUMENT 44 +#define HAVE_ARGUMENT 43 #define MIN_INSTRUMENTED_OPCODE 236 #ifdef __cplusplus diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index b3d7b8103e86c4..4da924bd250821 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -211,66 +211,66 @@ 'GET_LEN': 20, 'GET_YIELD_FROM_ITER': 21, 'INTERPRETER_EXIT': 22, - 'LOAD_ASSERTION_ERROR': 23, - 'LOAD_BUILD_CLASS': 24, - 'LOAD_LOCALS': 25, - 'MAKE_FUNCTION': 26, - 'MATCH_KEYS': 27, - 'MATCH_MAPPING': 28, - 'MATCH_SEQUENCE': 29, - 'NOP': 30, - 'POP_EXCEPT': 31, - 'POP_TOP': 32, - 'PUSH_EXC_INFO': 33, - 'PUSH_NULL': 34, - 'RETURN_GENERATOR': 35, - 'RETURN_VALUE': 36, - 'SETUP_ANNOTATIONS': 37, - 'STORE_SLICE': 38, - 'STORE_SUBSCR': 39, - 'TO_BOOL': 40, - 'UNARY_INVERT': 41, - 'UNARY_NEGATIVE': 42, - 'UNARY_NOT': 43, - 'WITH_EXCEPT_START': 44, - 'BINARY_OP': 45, - 'BUILD_CONST_KEY_MAP': 46, - 'BUILD_LIST': 47, - 'BUILD_MAP': 48, - 'BUILD_SET': 49, - 'BUILD_SLICE': 50, - 'BUILD_STRING': 51, - 'BUILD_TUPLE': 52, - 'CALL': 53, - 'CALL_FUNCTION_EX': 54, - 'CALL_INTRINSIC_1': 55, - 'CALL_INTRINSIC_2': 56, - 'CALL_KW': 57, - 'COMPARE_OP': 58, - 'CONTAINS_OP': 59, - 'CONVERT_VALUE': 60, - 'COPY': 61, - 'COPY_FREE_VARS': 62, - 'DELETE_ATTR': 63, - 'DELETE_DEREF': 64, - 'DELETE_FAST': 65, - 'DELETE_GLOBAL': 66, - 'DELETE_NAME': 67, - 'DICT_MERGE': 68, - 'DICT_UPDATE': 69, - 'ENTER_EXECUTOR': 70, - 'EXTENDED_ARG': 71, - 'FOR_ITER': 72, - 'GET_AWAITABLE': 73, - 'IMPORT_FROM': 74, - 'IMPORT_NAME': 75, - 'IS_OP': 76, - 'JUMP_BACKWARD': 77, - 'JUMP_BACKWARD_NO_INTERRUPT': 78, - 'JUMP_FORWARD': 79, - 'LIST_APPEND': 80, - 'LIST_EXTEND': 81, - 'LOAD_ATTR': 82, + 'LOAD_BUILD_CLASS': 23, + 'LOAD_LOCALS': 24, + 'MAKE_FUNCTION': 25, + 'MATCH_KEYS': 26, + 'MATCH_MAPPING': 27, + 'MATCH_SEQUENCE': 28, + 'NOP': 29, + 'POP_EXCEPT': 30, + 'POP_TOP': 31, + 'PUSH_EXC_INFO': 32, + 'PUSH_NULL': 33, + 'RETURN_GENERATOR': 34, + 'RETURN_VALUE': 35, + 'SETUP_ANNOTATIONS': 36, + 'STORE_SLICE': 37, + 'STORE_SUBSCR': 38, + 'TO_BOOL': 39, + 'UNARY_INVERT': 40, + 'UNARY_NEGATIVE': 41, + 'UNARY_NOT': 42, + 'WITH_EXCEPT_START': 43, + 'BINARY_OP': 44, + 'BUILD_CONST_KEY_MAP': 45, + 'BUILD_LIST': 46, + 'BUILD_MAP': 47, + 'BUILD_SET': 48, + 'BUILD_SLICE': 49, + 'BUILD_STRING': 50, + 'BUILD_TUPLE': 51, + 'CALL': 52, + 'CALL_FUNCTION_EX': 53, + 'CALL_INTRINSIC_1': 54, + 'CALL_INTRINSIC_2': 55, + 'CALL_KW': 56, + 'COMPARE_OP': 57, + 'CONTAINS_OP': 58, + 'CONVERT_VALUE': 59, + 'COPY': 60, + 'COPY_FREE_VARS': 61, + 'DELETE_ATTR': 62, + 'DELETE_DEREF': 63, + 'DELETE_FAST': 64, + 'DELETE_GLOBAL': 65, + 'DELETE_NAME': 66, + 'DICT_MERGE': 67, + 'DICT_UPDATE': 68, + 'ENTER_EXECUTOR': 69, + 'EXTENDED_ARG': 70, + 'FOR_ITER': 71, + 'GET_AWAITABLE': 72, + 'IMPORT_FROM': 73, + 'IMPORT_NAME': 74, + 'IS_OP': 75, + 'JUMP_BACKWARD': 76, + 'JUMP_BACKWARD_NO_INTERRUPT': 77, + 'JUMP_FORWARD': 78, + 'LIST_APPEND': 79, + 'LIST_EXTEND': 80, + 'LOAD_ATTR': 81, + 'LOAD_COMMON_CONSTANT': 82, 'LOAD_CONST': 83, 'LOAD_DEREF': 84, 'LOAD_FAST': 85, @@ -339,5 +339,5 @@ 'STORE_FAST_MAYBE_NULL': 267, } -HAVE_ARGUMENT = 44 +HAVE_ARGUMENT = 43 MIN_INSTRUMENTED_OPCODE = 236 diff --git a/Lib/dis.py b/Lib/dis.py index 76934eb00e63f0..f5bb7976b5fa62 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -11,6 +11,7 @@ _cache_format, _inline_cache_entries, _nb_ops, + _common_constants, _intrinsic_1_descs, _intrinsic_2_descs, _specializations, @@ -44,6 +45,7 @@ LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] +LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] LOAD_FAST_LOAD_FAST = opmap['LOAD_FAST_LOAD_FAST'] STORE_FAST_LOAD_FAST = opmap['STORE_FAST_LOAD_FAST'] STORE_FAST_STORE_FAST = opmap['STORE_FAST_STORE_FAST'] @@ -601,6 +603,12 @@ def get_argval_argrepr(self, op, arg, offset): argrepr = _intrinsic_1_descs[arg] elif deop == CALL_INTRINSIC_2: argrepr = _intrinsic_2_descs[arg] + elif deop == LOAD_COMMON_CONSTANT: + obj = _common_constants[arg] + if isinstance(obj, type): + argrepr = obj.__name__ + else: + argrepr = repr(obj) return argval, argrepr def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False): diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index db446776901fc3..b3abf380a82b11 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -472,8 +472,9 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3568 (Change semantics of END_FOR) # Python 3.13a5 3569 (Specialize CONTAINS_OP) # Python 3.13a6 3570 (Add __firstlineno__ class attribute) +# Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT) -# Python 3.14 will start with 3600 +# Python 3.15 will start with 3700 # Please don't copy-paste the same pre-release tag for new entries above!!! # You should always use the *upcoming* tag. For example, if 3.12a6 came out @@ -488,7 +489,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3570).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3571).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 5735686fa7fb44..85e37ff53e577f 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -36,6 +36,7 @@ _intrinsic_1_descs = _opcode.get_intrinsic1_descs() _intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_common_constants = [AssertionError, NotImplementedError] _nb_ops = _opcode.get_nb_ops() hascompare = [opmap["COMPARE_OP"]] diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 059fab1e2ecc08..ba77e1c5341db8 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -783,7 +783,7 @@ def f(): co_code=bytes( [ dis.opmap["RESUME"], 0, - dis.opmap["LOAD_ASSERTION_ERROR"], 0, + dis.opmap["LOAD_COMMON_CONSTANT"], 0, dis.opmap["RAISE_VARARGS"], 1, ] ), diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 1f4368b15f473c..ba0bcc9c1ced99 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1538,7 +1538,7 @@ def test_multiline_assert(self): ccc == 1000000), "error msg" """) compiled_code, _ = self.check_positions_against_ast(snippet) - self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_ASSERTION_ERROR', + self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_COMMON_CONSTANT', line=1, end_line=3, column=0, end_column=36, occurrence=1) # The "error msg": self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_CONST', diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index b68ed3baadc652..67a630e1346109 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -176,7 +176,7 @@ def bug1333982(x=[]): dis_bug1333982 = """\ %3d RESUME 0 -%3d LOAD_ASSERTION_ERROR +%3d LOAD_COMMON_CONSTANT 0 (AssertionError) LOAD_CONST 1 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION LOAD_FAST 0 (x) @@ -1574,190 +1574,190 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=52, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=26, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='BUILD_LIST', opcode=47, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='BUILD_MAP', opcode=48, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None), - Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=51, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_FUNCTION', opcode=25, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=110, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=84, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=84, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_LIST', opcode=46, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_MAP', opcode=47, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=62, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=52, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=26, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None), - Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=61, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=51, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_FUNCTION', opcode=25, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=110, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=84, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=84, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=84, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=84, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=85, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=35, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=62, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=88, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=61, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=84, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=84, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=84, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=84, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=88, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='GET_ITER', opcode=19, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='FOR_ITER', opcode=72, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='FOR_ITER', opcode=71, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='COMPARE_OP', opcode=57, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=76, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='COMPARE_OP', opcode=57, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=76, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=78, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), Instruction(opname='END_FOR', opcode=11, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_FAST_CHECK', opcode=87, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=40, argval=208, argrepr='to L9', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=6, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=45, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='BINARY_OP', opcode=44, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='COMPARE_OP', opcode=57, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=176, argrepr='to L7', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=76, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=7, positions=None, cache_info=None), Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='COMPARE_OP', opcode=57, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=1, argval=190, argrepr='to L8', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=20, argval=230, argrepr='to L10', offset=188, start_offset=188, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=78, arg=20, argval=230, argrepr='to L10', offset=188, start_offset=188, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=190, start_offset=190, starts_line=True, line_number=11, label=8, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=192, start_offset=192, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=192, start_offset=192, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=208, argrepr='to L9', offset=200, start_offset=200, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=40, argval=128, argrepr='to L6', offset=204, start_offset=204, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=76, arg=40, argval=128, argrepr='to L6', offset=204, start_offset=204, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=208, start_offset=208, starts_line=True, line_number=19, label=9, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_CONST', opcode=83, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='NOP', opcode=30, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=True, line_number=20, label=10, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=29, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=True, line_number=20, label=10, positions=None, cache_info=None), Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=232, start_offset=232, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_CONST', opcode=83, arg=7, argval=0, argrepr='0', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=45, arg=11, argval=11, argrepr='/', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=44, arg=11, argval=11, argrepr='/', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=242, start_offset=242, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='STORE_FAST', opcode=110, arg=1, argval='dodgy', argrepr='dodgy', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=248, start_offset=248, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_CONST', opcode=83, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=258, start_offset=258, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=270, start_offset=270, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=2, argval=2, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=2, argval=2, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=286, start_offset=286, starts_line=True, line_number=28, label=11, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=296, start_offset=296, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=306, start_offset=306, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=306, start_offset=306, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=308, start_offset=308, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=310, start_offset=310, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='WITH_EXCEPT_START', opcode=44, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=310, start_offset=310, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=43, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=39, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=1, argval=328, argrepr='to L12', offset=322, start_offset=322, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=12, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=26, argval=286, argrepr='to L11', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=338, start_offset=338, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=12, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=30, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=77, arg=26, argval=286, argrepr='to L11', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=60, arg=3, argval=3, argrepr='', offset=338, start_offset=338, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=30, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_GLOBAL', opcode=91, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=346, start_offset=346, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='CHECK_EXC_MATCH', opcode=7, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=14, argval=390, argrepr='to L13', offset=358, start_offset=358, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=364, start_offset=364, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_CONST', opcode=83, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=374, start_offset=374, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=52, argval=286, argrepr='to L11', offset=388, start_offset=388, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=30, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=77, arg=52, argval=286, argrepr='to L11', offset=388, start_offset=388, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=390, start_offset=390, starts_line=True, line_number=22, label=13, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=392, start_offset=392, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=394, start_offset=394, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=60, arg=3, argval=3, argrepr='', offset=392, start_offset=392, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=30, arg=None, argval=None, argrepr='', offset=394, start_offset=394, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=32, arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=410, start_offset=410, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=31, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=424, start_offset=424, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=60, arg=3, argval=3, argrepr='', offset=424, start_offset=424, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=30, arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-21-12-17-02.gh-issue-119180.UNDUb9.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-12-17-02.gh-issue-119180.UNDUb9.rst new file mode 100644 index 00000000000000..54c6639e0fa524 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-12-17-02.gh-issue-119180.UNDUb9.rst @@ -0,0 +1,2 @@ +Replace :opcode:`!LOAD_ASSERTION_ERROR` opcode with :opcode:`LOAD_COMMON_CONSTANT` +and add support for :exc:`NotImplementedError`. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 1c6b55643fa19e..2ffda8dee07c90 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -863,7 +863,7 @@ PyUnstable_Code_New(int argcount, int kwonlyargcount, static const uint8_t assert0[6] = { RESUME, RESUME_AT_FUNC_START, - LOAD_ASSERTION_ERROR, 0, + LOAD_COMMON_CONSTANT, CONSTANT_ASSERTIONERROR, RAISE_VARARGS, 1 }; diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 3e7ab7bffb158e..cdc417e48ebec6 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -2,16 +2,16 @@ unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, 0,0,0,0,0,243,166,0,0,0,149,0,83,0,83,1, - 75,0,114,0,83,0,83,1,75,1,114,1,92,2,34,0, - 83,2,53,1,0,0,0,0,0,0,32,0,92,2,34,0, - 83,3,92,0,82,6,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,53,2,0,0,0,0,0,0, - 32,0,92,1,82,8,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,34,0,53,0,0,0,0,0, - 0,0,83,4,5,0,0,0,114,5,83,5,19,0,72,20, - 0,0,114,6,92,2,34,0,83,6,92,6,14,0,83,7, - 92,5,92,6,5,0,0,0,14,0,51,4,53,1,0,0, - 0,0,0,0,32,0,77,22,0,0,11,0,32,0,103,1, + 74,0,114,0,83,0,83,1,74,1,114,1,92,2,33,0, + 83,2,52,1,0,0,0,0,0,0,31,0,92,2,33,0, + 83,3,92,0,81,6,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,52,2,0,0,0,0,0,0, + 31,0,92,1,81,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,33,0,52,0,0,0,0,0, + 0,0,83,4,5,0,0,0,114,5,83,5,19,0,71,20, + 0,0,114,6,92,2,33,0,83,6,92,6,14,0,83,7, + 92,5,92,6,5,0,0,0,14,0,50,4,52,1,0,0, + 0,0,0,0,31,0,76,22,0,0,11,0,31,0,103,1, 41,8,233,0,0,0,0,78,122,18,70,114,111,122,101,110, 32,72,101,108,108,111,32,87,111,114,108,100,122,8,115,121, 115,46,97,114,103,118,218,6,99,111,110,102,105,103,41,5, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 434eb804213810..b48f913b456064 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1184,8 +1184,18 @@ dummy_func( } } - inst(LOAD_ASSERTION_ERROR, ( -- value)) { - value = Py_NewRef(PyExc_AssertionError); + inst(LOAD_COMMON_CONSTANT, ( -- value)) { + // Keep in sync with _common_constants in opcode.py + switch(oparg) { + case CONSTANT_ASSERTIONERROR: + value = PyExc_AssertionError; + break; + case CONSTANT_NOTIMPLEMENTEDERROR: + value = PyExc_NotImplementedError; + break; + default: + Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); + } } inst(LOAD_BUILD_CLASS, ( -- bc)) { diff --git a/Python/compile.c b/Python/compile.c index 79f3baadca6b4a..6dacfc7cb55aa6 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3974,7 +3974,7 @@ compiler_assert(struct compiler *c, stmt_ty s) } NEW_JUMP_TARGET_LABEL(c, end); RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.Assert.test, end, 1)); - ADDOP(c, LOC(s), LOAD_ASSERTION_ERROR); + ADDOP_I(c, LOC(s), LOAD_COMMON_CONSTANT, CONSTANT_ASSERTIONERROR); if (s->v.Assert.msg) { VISIT(c, expr, s->v.Assert.msg); ADDOP_I(c, LOC(s), CALL, 0); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 347a1e677a0832..a3d7af250316e3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1162,9 +1162,20 @@ break; } - case _LOAD_ASSERTION_ERROR: { + case _LOAD_COMMON_CONSTANT: { PyObject *value; - value = Py_NewRef(PyExc_AssertionError); + oparg = CURRENT_OPARG(); + // Keep in sync with _common_constants in opcode.py + switch(oparg) { + case CONSTANT_ASSERTIONERROR: + value = PyExc_AssertionError; + break; + case CONSTANT_NOTIMPLEMENTEDERROR: + value = PyExc_NotImplementedError; + break; + default: + Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); + } stack_pointer[0] = value; stack_pointer += 1; break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 96161c5a6586fd..07d9965b299e12 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3695,17 +3695,6 @@ DISPATCH(); } - TARGET(LOAD_ASSERTION_ERROR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - TARGET(LOAD_ATTR) { frame->instr_ptr = next_instr; next_instr += 10; @@ -4251,6 +4240,27 @@ DISPATCH(); } + TARGET(LOAD_COMMON_CONSTANT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + PyObject *value; + // Keep in sync with _common_constants in opcode.py + switch(oparg) { + case CONSTANT_ASSERTIONERROR: + value = PyExc_AssertionError; + break; + case CONSTANT_NOTIMPLEMENTEDERROR: + value = PyExc_NotImplementedError; + break; + default: + Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); + } + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + TARGET(LOAD_CONST) { frame->instr_ptr = next_instr; next_instr += 1; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index fa4f1f8cbb475a..322483fefecf91 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -22,7 +22,6 @@ static void *opcode_targets[256] = { &&TARGET_GET_LEN, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_INTERPRETER_EXIT, - &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, &&TARGET_MAKE_FUNCTION, @@ -82,6 +81,7 @@ static void *opcode_targets[256] = { &&TARGET_LIST_APPEND, &&TARGET_LIST_EXTEND, &&TARGET_LOAD_ATTR, + &&TARGET_LOAD_COMMON_CONSTANT, &&TARGET_LOAD_CONST, &&TARGET_LOAD_DEREF, &&TARGET_LOAD_FAST, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index cd4431d55ad908..30ed011a83eb6e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -628,7 +628,7 @@ break; } - case _LOAD_ASSERTION_ERROR: { + case _LOAD_COMMON_CONSTANT: { _Py_UopsSymbol *value; value = sym_new_not_null(ctx); stack_pointer[0] = value; From 0e3c8cda1f04c983994e76aea93600dbb4714832 Mon Sep 17 00:00:00 2001 From: Landon Wood Date: Tue, 21 May 2024 21:15:40 -0400 Subject: [PATCH 160/903] gh-110383: Align dict.get(), .fromkeys(), and .setdefault() docs with docstrings (#119330) --- Doc/library/stdtypes.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index f97597dc565449..c0a3d0b3a2a49e 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4565,7 +4565,7 @@ can be used interchangeably to index the same dictionary entry. Return a shallow copy of the dictionary. - .. classmethod:: fromkeys(iterable[, value]) + .. classmethod:: fromkeys(iterable, value=None) Create a new dictionary with keys from *iterable* and values set to *value*. @@ -4575,7 +4575,7 @@ can be used interchangeably to index the same dictionary entry. such as an empty list. To get distinct values, use a :ref:`dict comprehension ` instead. - .. method:: get(key[, default]) + .. method:: get(key, default=None) Return the value for *key* if *key* is in the dictionary, else *default*. If *default* is not given, it defaults to ``None``, so that this method @@ -4617,7 +4617,7 @@ can be used interchangeably to index the same dictionary entry. .. versionadded:: 3.8 - .. method:: setdefault(key[, default]) + .. method:: setdefault(key, default=None) If *key* is in the dictionary, return its value. If not, insert *key* with a value of *default* and return *default*. *default* defaults to From d065edfb66470bbf06367b3570661d0346aa6707 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 22 May 2024 04:39:26 +0300 Subject: [PATCH 161/903] gh-60191: Implement ast.compare (#19211) * bpo-15987: Implement ast.compare Add a compare() function that compares two ASTs for structural equality. There are two set of attributes on AST node objects, fields and attributes. The fields are always compared, since they represent the actual structure of the code. The attributes can be optionally be included in the comparison. Attributes capture things like line numbers of column offsets, so comparing them involves test whether the layout of the program text is the same. Since whitespace seems inessential for comparing ASTs, the default is to compare fields but not attributes. ASTs are just Python objects that can be modified in arbitrary ways. The API for ASTs is under-specified in the presence of user modifications to objects. The comparison respects modifications to fields and attributes, and to _fields and _attributes attributes. A user could create obviously malformed objects, and the code will probably fail with an AttributeError when that happens. (For example, adding "spam" to _fields but not adding a "spam" attribute to the object.) Co-authored-by: Jeremy Hylton --- Doc/library/ast.rst | 14 ++ Doc/whatsnew/3.14.rst | 7 + Lib/ast.py | 71 ++++++++++ Lib/test/test_ast.py | 121 +++++++++++++++++- .../2020-03-28-21-00-54.bpo-15987.aBL8XS.rst | 2 + 5 files changed, 210 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-03-28-21-00-54.bpo-15987.aBL8XS.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index d4ccf282a5d00a..9ee56b92431b57 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2472,6 +2472,20 @@ effects on the compilation of a program: .. versionadded:: 3.8 +.. function:: compare(a, b, /, *, compare_attributes=False) + + Recursively compares two ASTs. + + *compare_attributes* affects whether AST attributes are considered + in the comparison. If *compare_attributes* is ``False`` (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + differ in whitespace or similar details. Attributes include line numbers + and column offsets. + + .. versionadded:: 3.14 + + .. _ast-cli: Command-Line Usage diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 27c985bec104fe..39172ac60cf1e0 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -86,6 +86,13 @@ New Modules Improved Modules ================ +ast +--- + +Added :func:`ast.compare` for comparing two ASTs. +(Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`) + + Optimizations ============= diff --git a/Lib/ast.py b/Lib/ast.py index d7e51aba595706..031bab43df7579 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -401,6 +401,77 @@ def walk(node): yield node +def compare( + a, + b, + /, + *, + compare_attributes=False, +): + """Recursively compares two ASTs. + + compare_attributes affects whether AST attributes are considered + in the comparison. If compare_attributes is False (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + might differ in whitespace or similar details. + """ + + def _compare(a, b): + # Compare two fields on an AST object, which may themselves be + # AST objects, lists of AST objects, or primitive ASDL types + # like identifiers and constants. + if isinstance(a, AST): + return compare( + a, + b, + compare_attributes=compare_attributes, + ) + elif isinstance(a, list): + # If a field is repeated, then both objects will represent + # the value as a list. + if len(a) != len(b): + return False + for a_item, b_item in zip(a, b): + if not _compare(a_item, b_item): + return False + else: + return True + else: + return type(a) is type(b) and a == b + + def _compare_fields(a, b): + if a._fields != b._fields: + return False + for field in a._fields: + a_field = getattr(a, field) + b_field = getattr(b, field) + if not _compare(a_field, b_field): + return False + else: + return True + + def _compare_attributes(a, b): + if a._attributes != b._attributes: + return False + # Attributes are always ints. + for attr in a._attributes: + a_attr = getattr(a, attr) + b_attr = getattr(b, attr) + if a_attr != b_attr: + return False + else: + return True + + if type(a) is not type(b): + return False + if not _compare_fields(a, b): + return False + if compare_attributes and not _compare_attributes(a, b): + return False + return True + + class NodeVisitor(object): """ A node visitor base class that walks the abstract syntax tree and calls a diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 5422c861ffb5c0..8a4374c56cbc08 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -38,6 +38,9 @@ def to_tuple(t): result.append(to_tuple(getattr(t, f))) return tuple(result) +STDLIB = os.path.dirname(ast.__file__) +STDLIB_FILES = [fn for fn in os.listdir(STDLIB) if fn.endswith(".py")] +STDLIB_FILES.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) # These tests are compiled through "exec" # There should be at least one test per statement @@ -1066,6 +1069,114 @@ def test_ast_asdl_signature(self): expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}" self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions) + def test_compare_basics(self): + self.assertTrue(ast.compare(ast.parse("x = 10"), ast.parse("x = 10"))) + self.assertFalse(ast.compare(ast.parse("x = 10"), ast.parse(""))) + self.assertFalse(ast.compare(ast.parse("x = 10"), ast.parse("x"))) + self.assertFalse( + ast.compare(ast.parse("x = 10;y = 20"), ast.parse("class C:pass")) + ) + + def test_compare_modified_ast(self): + # The ast API is a bit underspecified. The objects are mutable, + # and even _fields and _attributes are mutable. The compare() does + # some simple things to accommodate mutability. + a = ast.parse("m * x + b", mode="eval") + b = ast.parse("m * x + b", mode="eval") + self.assertTrue(ast.compare(a, b)) + + a._fields = a._fields + ("spam",) + a.spam = "Spam" + self.assertNotEqual(a._fields, b._fields) + self.assertFalse(ast.compare(a, b)) + self.assertFalse(ast.compare(b, a)) + + b._fields = a._fields + b.spam = a.spam + self.assertTrue(ast.compare(a, b)) + self.assertTrue(ast.compare(b, a)) + + b._attributes = b._attributes + ("eggs",) + b.eggs = "eggs" + self.assertNotEqual(a._attributes, b._attributes) + self.assertFalse(ast.compare(a, b, compare_attributes=True)) + self.assertFalse(ast.compare(b, a, compare_attributes=True)) + + a._attributes = b._attributes + a.eggs = b.eggs + self.assertTrue(ast.compare(a, b, compare_attributes=True)) + self.assertTrue(ast.compare(b, a, compare_attributes=True)) + + def test_compare_literals(self): + constants = ( + -20, + 20, + 20.0, + 1, + 1.0, + True, + 0, + False, + frozenset(), + tuple(), + "ABCD", + "abcd", + "中文字", + 1e1000, + -1e1000, + ) + for next_index, constant in enumerate(constants[:-1], 1): + next_constant = constants[next_index] + with self.subTest(literal=constant, next_literal=next_constant): + self.assertTrue( + ast.compare(ast.Constant(constant), ast.Constant(constant)) + ) + self.assertFalse( + ast.compare( + ast.Constant(constant), ast.Constant(next_constant) + ) + ) + + same_looking_literal_cases = [ + {1, 1.0, True, 1 + 0j}, + {0, 0.0, False, 0 + 0j}, + ] + for same_looking_literals in same_looking_literal_cases: + for literal in same_looking_literals: + for same_looking_literal in same_looking_literals - {literal}: + self.assertFalse( + ast.compare( + ast.Constant(literal), + ast.Constant(same_looking_literal), + ) + ) + + def test_compare_fieldless(self): + self.assertTrue(ast.compare(ast.Add(), ast.Add())) + self.assertFalse(ast.compare(ast.Sub(), ast.Add())) + + def test_compare_modes(self): + for mode, sources in ( + ("exec", exec_tests), + ("eval", eval_tests), + ("single", single_tests), + ): + for source in sources: + a = ast.parse(source, mode=mode) + b = ast.parse(source, mode=mode) + self.assertTrue( + ast.compare(a, b), f"{ast.dump(a)} != {ast.dump(b)}" + ) + + def test_compare_attributes_option(self): + def parse(a, b): + return ast.parse(a), ast.parse(b) + + a, b = parse("2 + 2", "2+2") + self.assertTrue(ast.compare(a, b)) + self.assertTrue(ast.compare(a, b, compare_attributes=False)) + self.assertFalse(ast.compare(a, b, compare_attributes=True)) + def test_positional_only_feature_version(self): ast.parse('def foo(x, /): ...', feature_version=(3, 8)) ast.parse('def bar(x=1, /): ...', feature_version=(3, 8)) @@ -1222,6 +1333,7 @@ def test_none_checks(self) -> None: for node, attr, source in tests: self.assert_none_check(node, attr, source) + class ASTHelpers_Test(unittest.TestCase): maxDiff = None @@ -2191,16 +2303,15 @@ def test_nameconstant(self): @support.requires_resource('cpu') def test_stdlib_validates(self): - stdlib = os.path.dirname(ast.__file__) - tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] - tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) - for module in tests: + for module in STDLIB_FILES: with self.subTest(module): - fn = os.path.join(stdlib, module) + fn = os.path.join(STDLIB, module) with open(fn, "r", encoding="utf-8") as fp: source = fp.read() mod = ast.parse(source, fn) compile(mod, fn, "exec") + mod2 = ast.parse(source, fn) + self.assertTrue(ast.compare(mod, mod2)) constant_1 = ast.Constant(1) pattern_1 = ast.MatchValue(constant_1) diff --git a/Misc/NEWS.d/next/Library/2020-03-28-21-00-54.bpo-15987.aBL8XS.rst b/Misc/NEWS.d/next/Library/2020-03-28-21-00-54.bpo-15987.aBL8XS.rst new file mode 100644 index 00000000000000..b906393449656d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-03-28-21-00-54.bpo-15987.aBL8XS.rst @@ -0,0 +1,2 @@ +Implemented :func:`ast.compare` for comparing two ASTs. Patch by Batuhan +Taskaya with some help from Jeremy Hylton. From c886bece3b3a49f8a0f188aecfc1d6ff89d281e6 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 21 May 2024 22:35:44 -0400 Subject: [PATCH 162/903] gh-111201: Add append to screen method to avoid recalculation (#119274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_pyrepl/commands.py | 5 ++++- Lib/_pyrepl/completing_reader.py | 16 +++++++-------- Lib/_pyrepl/reader.py | 34 ++++++++++++++++++++++++++++---- Lib/_pyrepl/unix_console.py | 2 +- 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 51c7afebede5a8..3d9722d1586c2a 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -358,7 +358,10 @@ def do(self) -> None: class self_insert(EditCommand): def do(self) -> None: r = self.reader - r.insert(self.event * r.get_arg()) + text = self.event * r.get_arg() + r.insert(text) + if len(text) == 1 and r.pos == len(r.buffer): + r.calc_screen = r.append_to_screen class insert_nl(EditCommand): diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 19fc06feaf3ced..3f8506bfe8f8f8 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -187,8 +187,8 @@ def do(self) -> None: if p: r.insert(p) if last_is_completer: - if not r.cmpltn_menu_vis: - r.cmpltn_menu_vis = 1 + if not r.cmpltn_menu_visible: + r.cmpltn_menu_visible = True r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, r.cmpltn_menu_end, r.use_brackets, r.sort_in_column) @@ -208,7 +208,7 @@ def do(self) -> None: commands.self_insert.do(self) - if r.cmpltn_menu_vis: + if r.cmpltn_menu_visible: stem = r.get_stem() if len(stem) < 1: r.cmpltn_reset() @@ -235,7 +235,7 @@ class CompletingReader(Reader): ### Instance variables cmpltn_menu: list[str] = field(init=False) - cmpltn_menu_vis: int = field(init=False) + cmpltn_menu_visible: bool = field(init=False) cmpltn_menu_end: int = field(init=False) cmpltn_menu_choices: list[str] = field(init=False) @@ -255,9 +255,9 @@ def after_command(self, cmd: Command) -> None: if not isinstance(cmd, (complete, self_insert)): self.cmpltn_reset() - def calc_screen(self) -> list[str]: - screen = super().calc_screen() - if self.cmpltn_menu_vis: + def calc_complete_screen(self) -> list[str]: + screen = super().calc_complete_screen() + if self.cmpltn_menu_visible: ly = self.lxy[1] screen[ly:ly] = self.cmpltn_menu self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) @@ -270,7 +270,7 @@ def finish(self) -> None: def cmpltn_reset(self) -> None: self.cmpltn_menu = [] - self.cmpltn_menu_vis = 0 + self.cmpltn_menu_visible = False self.cmpltn_menu_end = 0 self.cmpltn_menu_choices = [] diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 2c8c9e7dc4b5da..9a207a241d5f8d 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -35,7 +35,9 @@ # types Command = commands.Command if False: + from typing import Callable from .types import Callback, SimpleContextManager, KeySpec, CommandName + CalcScreen = Callable[[], list[str]] def disp_str(buffer: str) -> tuple[str, list[int]]: @@ -231,9 +233,11 @@ class Reader: keymap: tuple[tuple[str, str], ...] = () input_trans: input.KeymapTranslator = field(init=False) input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) + screen: list[str] = field(default_factory=list) screeninfo: list[tuple[int, list[int]]] = field(init=False) cxy: tuple[int, int] = field(init=False) lxy: tuple[int, int] = field(init=False) + calc_screen: CalcScreen = field(init=False) def __post_init__(self) -> None: # Enable the use of `insert` without a `prepare` call - necessary to @@ -243,14 +247,36 @@ def __post_init__(self) -> None: self.input_trans = input.KeymapTranslator( self.keymap, invalid_cls="invalid-key", character_cls="self-insert" ) - self.screeninfo = [(0, [0])] + self.screeninfo = [(0, [])] self.cxy = self.pos2xy() self.lxy = (self.pos, 0) + self.calc_screen = self.calc_complete_screen def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: return default_keymap - def calc_screen(self) -> list[str]: + def append_to_screen(self) -> list[str]: + new_screen = self.screen.copy() or [''] + + new_character = self.buffer[-1] + new_character_len = wlen(new_character) + + last_line_len = wlen(new_screen[-1]) + if last_line_len + new_character_len >= self.console.width: # We need to wrap here + new_screen[-1] += '\\' + self.screeninfo[-1][1].append(1) + new_screen.append(self.buffer[-1]) + self.screeninfo.append((0, [new_character_len])) + else: + new_screen[-1] += self.buffer[-1] + self.screeninfo[-1][1].append(new_character_len) + self.cxy = self.pos2xy() + + # Reset the function that is used for completing the screen + self.calc_screen = self.calc_complete_screen + return new_screen + + def calc_complete_screen(self) -> list[str]: """The purpose of this method is to translate changes in self.buffer into changes in self.screen. Currently it rips everything down and starts from scratch, which whilst not @@ -563,8 +589,8 @@ def update_screen(self) -> None: def refresh(self) -> None: """Recalculate and refresh the screen.""" # this call sets up self.cxy, so call it first. - screen = self.calc_screen() - self.console.refresh(screen, self.cxy) + self.screen = self.calc_screen() + self.console.refresh(self.screen, self.cxy) self.dirty = False def do_cmd(self, cmd: tuple[str, list[str]]) -> None: diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 7c59f48df406e6..ec7d0636b9aeb3 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -293,7 +293,7 @@ def refresh(self, screen, c_xy): self.__show_cursor() - self.screen = screen + self.screen = screen.copy() self.move_cursor(cx, cy) self.flushoutput() From 73ab83b27f105a4509046ce26e35f20d66625195 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Tue, 21 May 2024 22:36:01 -0400 Subject: [PATCH 163/903] gh-119357: Increase test coverage for keymap in _pyrepl (#119358) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_pyrepl/completing_reader.py | 2 +- Lib/_pyrepl/keymap.py | 63 ++++++++++------------ Lib/test/test_pyrepl/test_keymap.py | 82 ++++++++++++++++++++++------- 3 files changed, 94 insertions(+), 53 deletions(-) diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 3f8506bfe8f8f8..c11d2dabdd2792 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -30,7 +30,7 @@ # types Command = commands.Command if False: - from .types import Callback, SimpleContextManager, KeySpec, CommandName + from .types import KeySpec, CommandName def prefix(wordlist: list[str], j: int = 0) -> str: diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py index a303589280e7f1..2fb03d1952382f 100644 --- a/Lib/_pyrepl/keymap.py +++ b/Lib/_pyrepl/keymap.py @@ -19,38 +19,32 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -functions for parsing keyspecs +Keymap contains functions for parsing keyspecs and turning keyspecs into +appropriate sequences. -Support for turning keyspecs into appropriate sequences. +A keyspec is a string representing a sequence of key presses that can +be bound to a command. All characters other than the backslash represent +themselves. In the traditional manner, a backslash introduces an escape +sequence. -pyrepl uses it's own bastardized keyspec format, which is meant to be -a strict superset of readline's \"KEYSEQ\" format (which is to say -that if you can come up with a spec readline accepts that this -doesn't, you've found a bug and should tell me about it). - -Note that this is the `\\C-o' style of readline keyspec, not the -`Control-o' sort. - -A keyspec is a string representing a sequence of keypresses that can -be bound to a command. - -All characters other than the backslash represent themselves. In the -traditional manner, a backslash introduces a escape sequence. +pyrepl uses its own keyspec format that is meant to be a strict superset of +readline's KEYSEQ format. This means that if a spec is found that readline +accepts that this doesn't, it should be logged as a bug. Note that this means +we're using the `\\C-o' style of readline's keyspec, not the `Control-o' sort. The extension to readline is that the sequence \\ denotes the -sequence of charaters produced by hitting KEY. +sequence of characters produced by hitting KEY. Examples: - -`a' - what you get when you hit the `a' key +`a' - what you get when you hit the `a' key `\\EOA' - Escape - O - A (up, on my terminal) `\\' - the up arrow key -`\\' - ditto (keynames are case insensitive) +`\\' - ditto (keynames are case-insensitive) `\\C-o', `\\c-o' - control-o `\\M-.' - meta-period `\\E.' - ditto (that's how meta works for pyrepl) `\\', `\\', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' - - all of these are the tab character. Can you think of any more? + - all of these are the tab character. """ _escapes = { @@ -111,7 +105,17 @@ class KeySpecError(Exception): pass -def _parse_key1(key, s): +def parse_keys(keys: str) -> list[str]: + """Parse keys in keyspec format to a sequence of keys.""" + s = 0 + r: list[str] = [] + while s < len(keys): + k, s = _parse_single_key_sequence(keys, s) + r.extend(k) + return r + + +def _parse_single_key_sequence(key: str, s: int) -> tuple[list[str], int]: ctrl = 0 meta = 0 ret = "" @@ -183,20 +187,11 @@ def _parse_key1(key, s): ret = f"ctrl {ret}" else: raise KeySpecError("\\C- followed by invalid key") - if meta: - ret = ["\033", ret] - else: - ret = [ret] - return ret, s - -def parse_keys(key: str) -> list[str]: - s = 0 - r = [] - while s < len(key): - k, s = _parse_key1(key, s) - r.extend(k) - return r + result = [ret], s + if meta: + result[0].insert(0, "\033") + return result def compile_keymap(keymap, empty=b""): diff --git a/Lib/test/test_pyrepl/test_keymap.py b/Lib/test/test_pyrepl/test_keymap.py index 419f1643030b9d..2c97066b2c7043 100644 --- a/Lib/test/test_pyrepl/test_keymap.py +++ b/Lib/test/test_pyrepl/test_keymap.py @@ -1,41 +1,78 @@ +import string import unittest -from _pyrepl.keymap import parse_keys, compile_keymap +from _pyrepl.keymap import _keynames, _escapes, parse_keys, compile_keymap, KeySpecError class TestParseKeys(unittest.TestCase): def test_single_character(self): - self.assertEqual(parse_keys("a"), ["a"]) - self.assertEqual(parse_keys("b"), ["b"]) - self.assertEqual(parse_keys("1"), ["1"]) + """Ensure that single ascii characters or single digits are parsed as single characters.""" + test_cases = [(key, [key]) for key in string.ascii_letters + string.digits] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_keynames(self): + """Ensure that keynames are parsed to their corresponding mapping. + + A keyname is expected to be of the following form: \\ such as \\ + which would get parsed as "left". + """ + test_cases = [(f"\\<{keyname}>", [parsed_keyname]) for keyname, parsed_keyname in _keynames.items()] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) def test_escape_sequences(self): - self.assertEqual(parse_keys("\\n"), ["\n"]) - self.assertEqual(parse_keys("\\t"), ["\t"]) - self.assertEqual(parse_keys("\\\\"), ["\\"]) - self.assertEqual(parse_keys("\\'"), ["'"]) - self.assertEqual(parse_keys('\\"'), ['"']) + """Ensure that escaping sequences are parsed to their corresponding mapping.""" + test_cases = [(f"\\{escape}", [parsed_escape]) for escape, parsed_escape in _escapes.items()] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) def test_control_sequences(self): - self.assertEqual(parse_keys("\\C-a"), ["\x01"]) - self.assertEqual(parse_keys("\\C-b"), ["\x02"]) - self.assertEqual(parse_keys("\\C-c"), ["\x03"]) + """Ensure that supported control sequences are parsed successfully.""" + keys = ["@", "[", "]", "\\", "^", "_", "\\", "\\"] + keys.extend(string.ascii_letters) + test_cases = [(f"\\C-{key}", chr(ord(key) & 0x1F)) for key in []] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) def test_meta_sequences(self): self.assertEqual(parse_keys("\\M-a"), ["\033", "a"]) self.assertEqual(parse_keys("\\M-b"), ["\033", "b"]) self.assertEqual(parse_keys("\\M-c"), ["\033", "c"]) - def test_keynames(self): - self.assertEqual(parse_keys("\\"), ["up"]) - self.assertEqual(parse_keys("\\"), ["down"]) - self.assertEqual(parse_keys("\\"), ["left"]) - self.assertEqual(parse_keys("\\"), ["right"]) - def test_combinations(self): self.assertEqual(parse_keys("\\C-a\\n\\"), ["\x01", "\n", "up"]) self.assertEqual(parse_keys("\\M-a\\t\\"), ["\033", "a", "\t", "down"]) + def test_keyspec_errors(self): + cases = [ + ("\\Ca", "\\C must be followed by `-'"), + ("\\ca", "\\C must be followed by `-'"), + ("\\C-\\C-", "doubled \\C-"), + ("\\Ma", "\\M must be followed by `-'"), + ("\\ma", "\\M must be followed by `-'"), + ("\\M-\\M-", "doubled \\M-"), + ("\\", "unrecognised keyname"), + ("\\大", "unknown backslash escape"), + ("\\C-\\", "\\C- followed by invalid key") + ] + for test_keys, expected_err in cases: + with self.subTest(f"{test_keys} should give error {expected_err}"): + with self.assertRaises(KeySpecError) as e: + parse_keys(test_keys) + self.assertIn(expected_err, str(e.exception)) + + def test_index_errors(self): + test_cases = ["\\", "\\C", "\\C-\\C"] + for test_keys in test_cases: + with self.assertRaises(IndexError): + parse_keys(test_keys) + class TestCompileKeymap(unittest.TestCase): def test_empty_keymap(self): @@ -72,3 +109,12 @@ def test_nested_multiple_keymaps(self): keymap = {b"a": {b"b": {b"c": "action"}}} result = compile_keymap(keymap) self.assertEqual(result, {b"a": {b"b": {b"c": "action"}}}) + + def test_clashing_definitions(self): + km = {b'a': 'c', b'a' + b'b': 'd'} + with self.assertRaises(KeySpecError): + compile_keymap(km) + + def test_non_bytes_key(self): + with self.assertRaises(TypeError): + compile_keymap({123: 'a'}) From e9875ecb5dd3a9c44a184c71cc562ce1fea6e03b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 21 May 2024 22:38:12 -0400 Subject: [PATCH 164/903] gh-119180: PEP 649: Add __annotate__ attributes (#119209) --- Include/cpython/funcobject.h | 1 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/test_sys.py | 2 +- Lib/test/test_type_annotations.py | 44 +++++++ Lib/test/test_typing.py | 2 +- Lib/typing.py | 1 + ...-05-20-10-10-51.gh-issue-119180.35xqpu.rst | 2 + Objects/funcobject.c | 64 +++++++++- Objects/moduleobject.c | 103 +++++++++++++-- Objects/typeobject.c | 117 +++++++++++++++++- 13 files changed, 324 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 5433ba48eefc69..598cd330bc9ca9 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -41,6 +41,7 @@ typedef struct { PyObject *func_weakreflist; /* List of weak references */ PyObject *func_module; /* The __module__ attribute, can be anything */ PyObject *func_annotations; /* Annotations, a dict or NULL */ + PyObject *func_annotate; /* Callable to fill the annotations dictionary */ PyObject *func_typeparams; /* Tuple of active type variables or NULL */ vectorcallfunc vectorcall; /* Version number for use by specializer. diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index ca7355b2b61aa7..33133aaaf00893 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -590,6 +590,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__all__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__and__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__anext__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__args__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__asyncio_running_event_loop__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index fbb25285f0f282..f5ea7b9bd7d433 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -79,6 +79,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__all__) STRUCT_FOR_ID(__and__) STRUCT_FOR_ID(__anext__) + STRUCT_FOR_ID(__annotate__) STRUCT_FOR_ID(__annotations__) STRUCT_FOR_ID(__args__) STRUCT_FOR_ID(__asyncio_running_event_loop__) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 508da40c53422d..c73408d6315312 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -588,6 +588,7 @@ extern "C" { INIT_ID(__all__), \ INIT_ID(__and__), \ INIT_ID(__anext__), \ + INIT_ID(__annotate__), \ INIT_ID(__annotations__), \ INIT_ID(__args__), \ INIT_ID(__asyncio_running_event_loop__), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index cc2fc15ac5cabf..d84c45a6b57887 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -78,6 +78,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__anext__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__annotate__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__annotations__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index ee3bd0092f9bf3..8fe1d77756866a 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1564,7 +1564,7 @@ def func(): check(x, size('3Pi2cP7P2ic??2P')) # function def func(): pass - check(func, size('15Pi')) + check(func, size('16Pi')) class c(): @staticmethod def foo(): diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 3dbb35afcb620f..5e3c3347a41571 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,4 +1,5 @@ import textwrap +import types import unittest from test.support import run_code @@ -212,3 +213,46 @@ def test_match(self): case 0: x: int = 1 """) + + +class AnnotateTests(unittest.TestCase): + """See PEP 649.""" + def test_manual_annotate(self): + def f(): + pass + mod = types.ModuleType("mod") + class X: + pass + + for obj in (f, mod, X): + with self.subTest(obj=obj): + self.check_annotations(obj) + + def check_annotations(self, f): + self.assertEqual(f.__annotations__, {}) + self.assertIs(f.__annotate__, None) + + with self.assertRaisesRegex(TypeError, "__annotate__ must be callable or None"): + f.__annotate__ = 42 + f.__annotate__ = lambda: 42 + with self.assertRaisesRegex(TypeError, r"takes 0 positional arguments but 1 was given"): + print(f.__annotations__) + + f.__annotate__ = lambda x: 42 + with self.assertRaisesRegex(TypeError, r"__annotate__ returned non-dict of type 'int'"): + print(f.__annotations__) + + f.__annotate__ = lambda x: {"x": x} + self.assertEqual(f.__annotations__, {"x": 1}) + + # Setting annotate to None does not invalidate the cached __annotations__ + f.__annotate__ = None + self.assertEqual(f.__annotations__, {"x": 1}) + + # But setting it to a new callable does + f.__annotate__ = lambda x: {"y": x} + self.assertEqual(f.__annotations__, {"y": 1}) + + # Setting f.__annotations__ also clears __annotate__ + f.__annotations__ = {"z": 43} + self.assertIs(f.__annotate__, None) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 64c4c497eb8934..dac55ceb9e99e0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3723,7 +3723,7 @@ def meth(self): pass acceptable_extra_attrs = { '_is_protocol', '_is_runtime_protocol', '__parameters__', - '__init__', '__annotations__', '__subclasshook__', + '__init__', '__annotations__', '__subclasshook__', '__annotate__', } self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) self.assertLessEqual( diff --git a/Lib/typing.py b/Lib/typing.py index 434574559e04fc..be49aa63464f05 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1889,6 +1889,7 @@ class _TypingEllipsis: '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', '__match_args__', '__static_attributes__', '__firstlineno__', + '__annotate__', }) # These special attributes will be not collected as protocol members. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst new file mode 100644 index 00000000000000..5a88ce097274fb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-20-10-10-51.gh-issue-119180.35xqpu.rst @@ -0,0 +1,2 @@ +Add an ``__annotate__`` attribute to functions, classes, and modules as part +of :pep:`649`. Patch by Jelle Zijlstra. diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 8a30213888ef87..4e78252052932c 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -124,6 +125,7 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_weakreflist = NULL; op->func_module = module; op->func_annotations = NULL; + op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = 0; @@ -202,6 +204,7 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_weakreflist = NULL; op->func_module = module; op->func_annotations = NULL; + op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = 0; @@ -512,7 +515,22 @@ static PyObject * func_get_annotation_dict(PyFunctionObject *op) { if (op->func_annotations == NULL) { - return NULL; + if (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate)) { + Py_RETURN_NONE; + } + PyObject *one = _PyLong_GetOne(); + PyObject *ann_dict = _PyObject_CallOneArg(op->func_annotate, one); + if (ann_dict == NULL) { + return NULL; + } + if (!PyDict_Check(ann_dict)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(ann_dict)->tp_name); + Py_DECREF(ann_dict); + return NULL; + } + Py_XSETREF(op->func_annotations, ann_dict); + return ann_dict; } if (PyTuple_CheckExact(op->func_annotations)) { PyObject *ann_tuple = op->func_annotations; @@ -565,7 +583,9 @@ PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) "non-dict annotations"); return -1; } - Py_XSETREF(((PyFunctionObject *)op)->func_annotations, annotations); + PyFunctionObject *func = (PyFunctionObject *)op; + Py_XSETREF(func->func_annotations, annotations); + Py_CLEAR(func->func_annotate); return 0; } @@ -763,10 +783,44 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignor return 0; } +static PyObject * +func_get_annotate(PyFunctionObject *op, void *Py_UNUSED(ignored)) +{ + if (op->func_annotate == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(op->func_annotate); +} + +static int +func_set_annotate(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "__annotate__ cannot be deleted"); + return -1; + } + if (Py_IsNone(value)) { + Py_XSETREF(op->func_annotate, value); + return 0; + } + else if (PyCallable_Check(value)) { + Py_XSETREF(op->func_annotate, Py_XNewRef(value)); + Py_CLEAR(op->func_annotations); + return 0; + } + else { + PyErr_SetString(PyExc_TypeError, + "__annotate__ must be callable or None"); + return -1; + } +} + static PyObject * func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) { - if (op->func_annotations == NULL) { + if (op->func_annotations == NULL && + (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate))) { op->func_annotations = PyDict_New(); if (op->func_annotations == NULL) return NULL; @@ -789,6 +843,7 @@ func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(igno return -1; } Py_XSETREF(op->func_annotations, Py_XNewRef(value)); + Py_CLEAR(op->func_annotate); return 0; } @@ -836,6 +891,7 @@ static PyGetSetDef func_getsetlist[] = { (setter)func_set_kwdefaults}, {"__annotations__", (getter)func_get_annotations, (setter)func_set_annotations}, + {"__annotate__", (getter)func_get_annotate, (setter)func_set_annotate}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, {"__name__", (getter)func_get_name, (setter)func_set_name}, {"__qualname__", (getter)func_get_qualname, (setter)func_set_qualname}, @@ -972,6 +1028,7 @@ func_clear(PyFunctionObject *op) Py_CLEAR(op->func_dict); Py_CLEAR(op->func_closure); Py_CLEAR(op->func_annotations); + Py_CLEAR(op->func_annotate); Py_CLEAR(op->func_typeparams); // Don't Py_CLEAR(op->func_code), since code is always required // to be non-NULL. Similarly, name and qualname shouldn't be NULL. @@ -1028,6 +1085,7 @@ func_traverse(PyFunctionObject *f, visitproc visit, void *arg) Py_VISIT(f->func_dict); Py_VISIT(f->func_closure); Py_VISIT(f->func_annotations); + Py_VISIT(f->func_annotate); Py_VISIT(f->func_typeparams); Py_VISIT(f->func_qualname); return 0; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 46995b948a28e7..73ad9711b6b0fc 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -5,6 +5,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_fileutils.h" // _Py_wgetcwd #include "pycore_interp.h" // PyInterpreterState.importlib +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_AllocNoTrack @@ -1133,7 +1134,7 @@ static PyMethodDef module_methods[] = { }; static PyObject * -module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) +module_get_dict(PyModuleObject *m) { PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); if (dict == NULL) { @@ -1144,10 +1145,97 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) Py_DECREF(dict); return NULL; } + return dict; +} + +static PyObject * +module_get_annotate(PyModuleObject *m, void *Py_UNUSED(ignored)) +{ + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return NULL; + } + + PyObject *annotate; + if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) == 0) { + annotate = Py_None; + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate) == -1) { + Py_CLEAR(annotate); + } + } + Py_DECREF(dict); + return annotate; +} + +static int +module_set_annotate(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); + return -1; + } + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return -1; + } + + if (!Py_IsNone(value) && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + Py_DECREF(dict); + return -1; + } + + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), value) == -1) { + Py_DECREF(dict); + return -1; + } + if (!Py_IsNone(value)) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + Py_DECREF(dict); + return -1; + } + } + Py_DECREF(dict); + return 0; +} + +static PyObject * +module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) +{ + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return NULL; + } PyObject *annotations; if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) == 0) { - annotations = PyDict_New(); + PyObject *annotate; + int annotate_result = PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate); + if (annotate_result < 0) { + Py_DECREF(dict); + return NULL; + } + if (annotate_result == 1 && PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + annotations = _PyObject_CallOneArg(annotate, one); + if (annotations == NULL) { + Py_DECREF(annotate); + Py_DECREF(dict); + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotate); + Py_DECREF(annotations); + Py_DECREF(dict); + return NULL; + } + } + else { + annotations = PyDict_New(); + } + Py_XDECREF(annotate); if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); @@ -1164,14 +1252,10 @@ static int module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) { int ret = -1; - PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); + PyObject *dict = module_get_dict(m); if (dict == NULL) { return -1; } - if (!PyDict_Check(dict)) { - PyErr_Format(PyExc_TypeError, ".__dict__ is not a dictionary"); - goto exit; - } if (value != NULL) { /* set */ @@ -1188,8 +1272,10 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor ret = 0; } } + if (ret == 0 && PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + ret = -1; + } -exit: Py_DECREF(dict); return ret; } @@ -1197,6 +1283,7 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor static PyGetSetDef module_getsets[] = { {"__annotations__", (getter)module_get_annotations, (setter)module_set_annotations}, + {"__annotate__", (getter)module_get_annotate, (setter)module_set_annotate}, {NULL} }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b7c3fcf47f23fc..9f000d8c193bc5 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7,7 +7,7 @@ #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_lock.h" // _PySeqLock_* -#include "pycore_long.h" // _PyLong_IsNegative() +#include "pycore_long.h" // _PyLong_IsNegative(), _PyLong_GetOne() #include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetDef() @@ -1674,6 +1674,76 @@ type_set_doc(PyTypeObject *type, PyObject *value, void *context) return PyDict_SetItem(dict, &_Py_ID(__doc__), value); } +static PyObject * +type_get_annotate(PyTypeObject *type, void *Py_UNUSED(ignored)) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '__annotate__'", type->tp_name); + return NULL; + } + + PyObject *annotate; + PyObject *dict = PyType_GetDict(type); + if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) < 0) { + Py_DECREF(dict); + return NULL; + } + if (annotate) { + descrgetfunc get = Py_TYPE(annotate)->tp_descr_get; + if (get) { + Py_SETREF(annotate, get(annotate, NULL, (PyObject *)type)); + } + } + else { + annotate = Py_None; + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate); + if (result < 0) { + Py_DECREF(dict); + return NULL; + } + } + Py_DECREF(dict); + return annotate; +} + +static int +type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored)) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); + return -1; + } + if (_PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)) { + PyErr_Format(PyExc_TypeError, + "cannot set '__annotate__' attribute of immutable type '%s'", + type->tp_name); + return -1; + } + + if (!Py_IsNone(value) && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + return -1; + } + + PyObject *dict = PyType_GetDict(type); + assert(PyDict_Check(dict)); + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), value); + if (result < 0) { + Py_DECREF(dict); + return -1; + } + if (!Py_IsNone(value)) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + Py_DECREF(dict); + PyType_Modified(type); + return -1; + } + } + Py_DECREF(dict); + PyType_Modified(type); + return 0; +} + static PyObject * type_get_annotations(PyTypeObject *type, void *context) { @@ -1683,8 +1753,9 @@ type_get_annotations(PyTypeObject *type, void *context) } PyObject *annotations; - PyObject *dict = lookup_tp_dict(type); + PyObject *dict = PyType_GetDict(type); if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) { + Py_DECREF(dict); return NULL; } if (annotations) { @@ -1694,7 +1765,32 @@ type_get_annotations(PyTypeObject *type, void *context) } } else { - annotations = PyDict_New(); + PyObject *annotate = type_get_annotate(type, NULL); + if (annotate == NULL) { + Py_DECREF(dict); + return NULL; + } + if (PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + annotations = _PyObject_CallOneArg(annotate, one); + if (annotations == NULL) { + Py_DECREF(dict); + Py_DECREF(annotate); + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotations); + Py_DECREF(annotate); + Py_DECREF(dict); + return NULL; + } + } + else { + annotations = PyDict_New(); + } + Py_DECREF(annotate); if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); @@ -1705,6 +1801,7 @@ type_get_annotations(PyTypeObject *type, void *context) } } } + Py_DECREF(dict); return annotations; } @@ -1719,7 +1816,7 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) } int result; - PyObject *dict = lookup_tp_dict(type); + PyObject *dict = PyType_GetDict(type); if (value != NULL) { /* set */ result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); @@ -1728,14 +1825,23 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); if (result == 0) { PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); return -1; } } if (result < 0) { + Py_DECREF(dict); return -1; } - + else if (result == 0) { + if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + PyType_Modified(type); + Py_DECREF(dict); + return -1; + } + } PyType_Modified(type); + Py_DECREF(dict); return 0; } @@ -1811,6 +1917,7 @@ static PyGetSetDef type_getsets[] = { {"__doc__", (getter)type_get_doc, (setter)type_set_doc, NULL}, {"__text_signature__", (getter)type_get_text_signature, NULL, NULL}, {"__annotations__", (getter)type_get_annotations, (setter)type_set_annotations, NULL}, + {"__annotate__", (getter)type_get_annotate, (setter)type_set_annotate, NULL}, {"__type_params__", (getter)type_get_type_params, (setter)type_set_type_params, NULL}, {0} }; From cd516cd1f5e94dba887353f421513fd172efadf3 Mon Sep 17 00:00:00 2001 From: Arnon Yaari Date: Wed, 22 May 2024 00:21:14 -0400 Subject: [PATCH 165/903] gh-111201: auto-indentation in _pyrepl (#119348) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_pyrepl/readline.py | 43 +++++- Lib/test/test_pyrepl/test_pyrepl.py | 194 ++++++++++++++++++++-------- 2 files changed, 179 insertions(+), 58 deletions(-) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 787dbc06e58e18..054a39b7442655 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -99,6 +99,7 @@ class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): # Instance fields config: ReadlineConfig more_lines: MoreLinesCallable | None = None + last_used_indentation: str | None = None def __post_init__(self) -> None: super().__post_init__() @@ -157,6 +158,11 @@ def get_trimmed_history(self, maxlength: int) -> list[str]: cut = 0 return self.history[cut:] + def update_last_used_indentation(self) -> None: + indentation = _get_first_indentation(self.buffer) + if indentation is not None: + self.last_used_indentation = indentation + # --- simplified support for reading multiline Python statements --- def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: @@ -211,6 +217,28 @@ def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | N return prevlinestart, indent +def _get_first_indentation(buffer: list[str]) -> str | None: + indented_line_start = None + for i in range(len(buffer)): + if (i < len(buffer) - 1 + and buffer[i] == "\n" + and buffer[i + 1] in " \t" + ): + indented_line_start = i + 1 + elif indented_line_start is not None and buffer[i] not in " \t\n": + return ''.join(buffer[indented_line_start : i]) + return None + + +def _is_last_char_colon(buffer: list[str]) -> bool: + i = len(buffer) + while i > 0: + i -= 1 + if buffer[i] not in " \t\n": # ignore whitespaces + return buffer[i] == ":" + return False + + class maybe_accept(commands.Command): def do(self) -> None: r: ReadlineAlikeReader @@ -227,9 +255,18 @@ def do(self) -> None: # auto-indent the next line like the previous line prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) r.insert("\n") - if not self.reader.paste_mode and indent: - for i in range(prevlinestart, prevlinestart + indent): - r.insert(r.buffer[i]) + if not self.reader.paste_mode: + if indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) + r.update_last_used_indentation() + if _is_last_char_colon(r.buffer): + if r.last_used_indentation is not None: + indentation = r.last_used_indentation + else: + # default + indentation = " " * 4 + r.insert(indentation) elif not self.reader.paste_mode: self.finish = True else: diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index b643ae5895c97e..930f6759fb0b48 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -5,19 +5,31 @@ from unittest import TestCase from unittest.mock import patch -from .support import FakeConsole, handle_all_events, handle_events_narrow_console -from .support import more_lines, multiline_input, code_to_events +from .support import ( + FakeConsole, + handle_all_events, + handle_events_narrow_console, + more_lines, + multiline_input, + code_to_events, +) from _pyrepl.console import Event from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig from _pyrepl.readline import multiline_input as readline_multiline_input class TestCursorPosition(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + def test_up_arrow_simple(self): # fmt: off code = ( - 'def f():\n' - ' ...\n' + "def f():\n" + " ...\n" ) # fmt: on events = itertools.chain( @@ -34,8 +46,8 @@ def test_up_arrow_simple(self): def test_down_arrow_end_of_input(self): # fmt: off code = ( - 'def f():\n' - ' ...\n' + "def f():\n" + " ...\n" ) # fmt: on events = itertools.chain( @@ -300,6 +312,79 @@ def test_cursor_position_after_wrap_and_move_up(self): self.assertEqual(reader.pos, 10) self.assertEqual(reader.cxy, (1, 1)) + def test_auto_indent_default(self): + # fmt: off + input_code = ( + "def f():\n" + "pass\n\n" + ) + + output_code = ( + "def f():\n" + " pass\n" + " " + ) + # fmt: on + + def test_auto_indent_continuation(self): + # auto indenting according to previous user indentation + # fmt: off + events = itertools.chain( + code_to_events("def f():\n"), + # add backspace to delete default auto-indent + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + code_to_events( + " pass\n" + "pass\n\n" + ), + ) + + output_code = ( + "def f():\n" + " pass\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_prev_block(self): + # auto indenting according to indentation in different block + # fmt: off + events = itertools.chain( + code_to_events("def f():\n"), + # add backspace to delete default auto-indent + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + code_to_events( + " pass\n" + "pass\n\n" + ), + code_to_events( + "def g():\n" + "pass\n\n" + ), + ) + + + output_code = ( + "def g():\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output1 = multiline_input(reader) + output2 = multiline_input(reader) + self.assertEqual(output2, output_code) + class TestPyReplOutput(TestCase): def prepare_reader(self, events): @@ -316,14 +401,12 @@ def test_basic(self): def test_multiline_edit(self): events = itertools.chain( - code_to_events("def f():\n ...\n\n"), + code_to_events("def f():\n...\n\n"), [ Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), Event(evt="key", data="g", raw=bytearray(b"g")), Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), @@ -334,9 +417,9 @@ def test_multiline_edit(self): reader = self.prepare_reader(events) output = multiline_input(reader) - self.assertEqual(output, "def f():\n ...\n ") + self.assertEqual(output, "def f():\n ...\n ") output = multiline_input(reader) - self.assertEqual(output, "def g():\n ...\n ") + self.assertEqual(output, "def g():\n ...\n ") def test_history_navigation_with_up_arrow(self): events = itertools.chain( @@ -485,6 +568,7 @@ class Dummy: @property def test_func(self): import warnings + warnings.warn("warnings\n") return None @@ -508,12 +592,12 @@ def prepare_reader(self, events): def test_paste(self): # fmt: off code = ( - 'def a():\n' - ' for x in range(10):\n' - ' if x%2:\n' - ' print(x)\n' - ' else:\n' - ' pass\n' + "def a():\n" + " for x in range(10):\n" + " if x%2:\n" + " print(x)\n" + " else:\n" + " pass\n" ) # fmt: on @@ -534,10 +618,10 @@ def test_paste(self): def test_paste_mid_newlines(self): # fmt: off code = ( - 'def f():\n' - ' x = y\n' - ' \n' - ' y = z\n' + "def f():\n" + " x = y\n" + " \n" + " y = z\n" ) # fmt: on @@ -558,16 +642,16 @@ def test_paste_mid_newlines(self): def test_paste_mid_newlines_not_in_paste_mode(self): # fmt: off code = ( - 'def f():\n' - ' x = y\n' - ' \n' - ' y = z\n\n' + "def f():\n" + "x = y\n" + "\n" + "y = z\n\n" ) expected = ( - 'def f():\n' - ' x = y\n' - ' ' + "def f():\n" + " x = y\n" + " " ) # fmt: on @@ -579,20 +663,20 @@ def test_paste_mid_newlines_not_in_paste_mode(self): def test_paste_not_in_paste_mode(self): # fmt: off input_code = ( - 'def a():\n' - ' for x in range(10):\n' - ' if x%2:\n' - ' print(x)\n' - ' else:\n' - ' pass\n\n' + "def a():\n" + "for x in range(10):\n" + "if x%2:\n" + "print(x)\n" + "else:\n" + "pass\n\n" ) output_code = ( - 'def a():\n' - ' for x in range(10):\n' - ' if x%2:\n' - ' print(x)\n' - ' else:' + "def a():\n" + " for x in range(10):\n" + " if x%2:\n" + " print(x)\n" + " else:" ) # fmt: on @@ -605,25 +689,25 @@ def test_bracketed_paste(self): """Test that bracketed paste using \x1b[200~ and \x1b[201~ works.""" # fmt: off input_code = ( - 'def a():\n' - ' for x in range(10):\n' - '\n' - ' if x%2:\n' - ' print(x)\n' - '\n' - ' else:\n' - ' pass\n' + "def a():\n" + " for x in range(10):\n" + "\n" + " if x%2:\n" + " print(x)\n" + "\n" + " else:\n" + " pass\n" ) output_code = ( - 'def a():\n' - ' for x in range(10):\n' - '\n' - ' if x%2:\n' - ' print(x)\n' - '\n' - ' else:\n' - ' pass\n' + "def a():\n" + " for x in range(10):\n" + "\n" + " if x%2:\n" + " print(x)\n" + "\n" + " else:\n" + " pass\n" ) # fmt: on From e6572e8f98d33994d2d0dd3afa92a2a72ee642a9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 22 May 2024 01:28:32 -0400 Subject: [PATCH 166/903] gh-111201: Speed up paste mode in the REPL (#119341) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_pyrepl/commands.py | 5 ++--- Lib/_pyrepl/reader.py | 8 ++++---- Lib/_pyrepl/readline.py | 11 ++++++----- Lib/_pyrepl/simple_interact.py | 3 ++- Lib/_pyrepl/utils.py | 8 ++++++-- Lib/test/test_pyrepl/test_pyrepl.py | 2 +- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 3d9722d1586c2a..33c6564f640421 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -461,8 +461,6 @@ def do(self) -> None: class paste_mode(Command): def do(self) -> None: - if not self.reader.paste_mode: - self.reader.was_paste_mode_activated = True self.reader.paste_mode = not self.reader.paste_mode self.reader.dirty = True @@ -470,9 +468,10 @@ def do(self) -> None: class enable_bracketed_paste(Command): def do(self) -> None: self.reader.paste_mode = True - self.reader.was_paste_mode_activated = True + self.reader.in_bracketed_paste = True class disable_bracketed_paste(Command): def do(self) -> None: self.reader.paste_mode = False + self.reader.in_bracketed_paste = False self.reader.dirty = True diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 9a207a241d5f8d..3dbfc34780e8e8 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -54,7 +54,7 @@ def disp_str(buffer: str) -> tuple[str, list[int]]: b: list[int] = [] s: list[str] = [] for c in buffer: - if unicodedata.category(c).startswith("C"): + if ord(c) > 128 and unicodedata.category(c).startswith("C"): c = r"\u%04x" % ord(c) s.append(c) b.append(wlen(c)) @@ -225,7 +225,7 @@ class Reader: dirty: bool = False finished: bool = False paste_mode: bool = False - was_paste_mode_activated: bool = False + in_bracketed_paste: bool = False commands: dict[str, type[Command]] = field(default_factory=make_default_commands) last_command: type[Command] | None = None syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) @@ -448,7 +448,7 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: elif "\n" in self.buffer: if lineno == 0: prompt = self.ps2 - elif lineno == self.buffer.count("\n"): + elif self.ps4 and lineno == self.buffer.count("\n"): prompt = self.ps4 else: prompt = self.ps3 @@ -611,7 +611,7 @@ def do_cmd(self, cmd: tuple[str, list[str]]) -> None: self.after_command(command) - if self.dirty: + if self.dirty and not self.in_bracketed_paste: self.refresh() else: self.update_cursor() diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 054a39b7442655..796f1ef86360de 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -328,7 +328,7 @@ def input(self, prompt: object = "") -> str: reader.ps1 = str(prompt) return reader.readline(startup_hook=self.startup_hook) - def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> tuple[str, bool]: + def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str: """Read an input on possibly multiple lines, asking for more lines as long as 'more_lines(unicodetext)' returns an object whose boolean value is true. @@ -337,14 +337,15 @@ def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> saved = reader.more_lines try: reader.more_lines = more_lines - reader.ps1 = reader.ps2 = ps1 - reader.ps3 = reader.ps4 = ps2 + reader.ps1 = ps1 + reader.ps2 = ps1 + reader.ps3 = ps2 + reader.ps4 = "" with warnings.catch_warnings(action="ignore"): - return reader.readline(), reader.was_paste_mode_activated + return reader.readline() finally: reader.more_lines = saved reader.paste_mode = False - reader.was_paste_mode_activated = False def parse_and_bind(self, string: str) -> None: pass # XXX we don't support parsing GNU-readline-style init files diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index d65b6d0d62790a..8ab4dab757685e 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -62,6 +62,7 @@ def _strip_final_indent(text: str) -> str: "quit": _sitebuiltins.Quitter('quit' ,''), "copyright": _sitebuiltins._Printer('copyright', sys.copyright), "help": "help", + "clear": "clear_screen", } class InteractiveColoredConsole(code.InteractiveConsole): @@ -163,7 +164,7 @@ def more_lines(unicodetext: str) -> bool: ps1 = getattr(sys, "ps1", ">>> ") ps2 = getattr(sys, "ps2", "... ") try: - statement, contains_pasted_code = multiline_input(more_lines, ps1, ps2) + statement = multiline_input(more_lines, ps1, ps2) except EOFError: break diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index cd1df7c49a216d..96e917e487d91a 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -1,10 +1,14 @@ import re import unicodedata +import functools ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") +@functools.cache def str_width(c: str) -> int: + if ord(c) < 128: + return 1 w = unicodedata.east_asian_width(c) if w in ('N', 'Na', 'H', 'A'): return 1 @@ -13,6 +17,6 @@ def str_width(c: str) -> int: def wlen(s: str) -> int: length = sum(str_width(i) for i in s) - # remove lengths of any escape sequences - return length - sum(len(i) for i in ANSI_ESCAPE_SEQUENCE.findall(s)) + sequence = ANSI_ESCAPE_SEQUENCE.findall(s) + return length - sum(len(i) for i in sequence) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 930f6759fb0b48..7b5217e4b01fd0 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -578,7 +578,7 @@ def test_func(self): reader = self.prepare_reader(events, namespace) mock_get_reader.return_value = reader output = readline_multiline_input(more_lines, ">>>", "...") - self.assertEqual(output[0], "dummy.test_func.__") + self.assertEqual(output, "dummy.test_func.__") self.assertEqual(mock_stderr.getvalue(), "") From 5091c4400c9ea2a2d1e4d89a28c9d0de2651fa6d Mon Sep 17 00:00:00 2001 From: Aya Elsayed Date: Wed, 22 May 2024 06:56:35 +0100 Subject: [PATCH 167/903] gh-118911: Trailing whitespace in a block shouldn't prevent the user from terminating the code block (#119355) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_pyrepl/historical_reader.py | 2 +- Lib/_pyrepl/readline.py | 17 ++++++- Lib/test/test_pyrepl/test_pyrepl.py | 19 +++++--- Lib/test/test_pyrepl/test_reader.py | 45 ++++++++++++++++++- ...-05-21-20-13-23.gh-issue-118911.iG8nMq.rst | 5 +++ 5 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-21-20-13-23.gh-issue-118911.iG8nMq.rst diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index eef7d901b083ef..121de33da5052f 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -259,7 +259,7 @@ def select_item(self, i: int) -> None: self.transient_history[self.historyi] = self.get_unicode() buf = self.transient_history.get(i) if buf is None: - buf = self.history[i] + buf = self.history[i].rstrip() self.buffer = list(buf) self.historyi = i self.pos = len(self.buffer) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 796f1ef86360de..ffa14a9ce31a8f 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -244,14 +244,27 @@ def do(self) -> None: r: ReadlineAlikeReader r = self.reader # type: ignore[assignment] r.dirty = True # this is needed to hide the completion menu, if visible - # + # if there are already several lines and the cursor # is not on the last one, always insert a new \n. text = r.get_unicode() + if "\n" in r.buffer[r.pos :] or ( r.more_lines is not None and r.more_lines(text) ): - # + def _newline_before_pos(): + before_idx = r.pos - 1 + while before_idx > 0 and text[before_idx].isspace(): + before_idx -= 1 + return text[before_idx : r.pos].count("\n") > 0 + + # if there's already a new line before the cursor then + # even if the cursor is followed by whitespace, we assume + # the user is trying to terminate the block + if _newline_before_pos() and text[r.pos:].isspace(): + self.finish = True + return + # auto-indent the next line like the previous line prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) r.insert("\n") diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 7b5217e4b01fd0..bdcabf9be05b9e 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -405,12 +405,21 @@ def test_multiline_edit(self): [ Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), - Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), - Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), Event(evt="key", data="g", raw=bytearray(b"g")), Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), - Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="delete", raw=bytearray(b"\x7F")), + Event(evt="key", data="right", raw=bytearray(b"g")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="p", raw=bytearray(b"p")), + Event(evt="key", data="a", raw=bytearray(b"a")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), Event(evt="key", data="\n", raw=bytearray(b"\n")), ], ) @@ -419,7 +428,7 @@ def test_multiline_edit(self): output = multiline_input(reader) self.assertEqual(output, "def f():\n ...\n ") output = multiline_input(reader) - self.assertEqual(output, "def g():\n ...\n ") + self.assertEqual(output, "def g():\n pass\n ") def test_history_navigation_with_up_arrow(self): events = itertools.chain( diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index dc7d8a5ba97cda..7bf7a36d8d7bb9 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -1,7 +1,8 @@ import itertools +import functools from unittest import TestCase -from .support import handle_all_events, handle_events_narrow_console, code_to_events +from .support import handle_all_events, handle_events_narrow_console, code_to_events, prepare_reader from _pyrepl.console import Event @@ -133,3 +134,45 @@ def test_up_arrow_after_ctrl_r(self): reader, _ = handle_all_events(events) self.assert_screen_equals(reader, "") + + def test_newline_within_block_trailing_whitespace(self): + # fmt: off + code = ( + "def foo():\n" + "a = 1\n" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + # go to the end of the first line + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # new lines in-block shouldn't terminate the block + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + # end of line 2 + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # a double new line in-block should terminate the block + # even if its followed by whitespace + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + no_paste_reader = functools.partial(prepare_reader, paste_mode=False) + reader, _ = handle_all_events(events, prepare_reader=no_paste_reader) + + expected = ( + "def foo():\n" + "\n" + "\n" + " a = 1\n" + " \n" + " " # HistoricalReader will trim trailing whitespace + ) + self.assert_screen_equals(reader, expected) + self.assertTrue(reader.finished) diff --git a/Misc/NEWS.d/next/Library/2024-05-21-20-13-23.gh-issue-118911.iG8nMq.rst b/Misc/NEWS.d/next/Library/2024-05-21-20-13-23.gh-issue-118911.iG8nMq.rst new file mode 100644 index 00000000000000..4f15c1b67c9774 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-21-20-13-23.gh-issue-118911.iG8nMq.rst @@ -0,0 +1,5 @@ +In PyREPL, updated ``maybe-accept``'s logic so that if the user hits +:kbd:`Enter` twice, they are able to terminate the block even if there's +trailing whitespace. Also, now when the user hits arrow up, the cursor +is on the last functional line. This matches IPython's behavior. +Patch by Aya Elsayed. From 31d61a75c9ae8c1b1bc6447f517122be8adce2ef Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Wed, 22 May 2024 16:38:06 +1000 Subject: [PATCH 168/903] DOCS: fix error in exec namespace note (gh-119378) When updating the new exec note added in gh-119235 as part of the PEP 667 general docs PR, I suggested a workaround that isn't valid. The first half of the note is still reasonable, so just omit the invalid text. --- Doc/library/functions.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index a879ddbca92e82..cb9b650badcfbd 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -623,10 +623,6 @@ are always available. They are listed here in alphabetical order. means functions and classes defined in the executed code will not be able to access variables assigned at the top level (as the "top level" variables are treated as class variables in a class definition). - Passing a :class:`collections.ChainMap` instance as *globals* allows name - lookups to be chained across multiple mappings without triggering this - behaviour. Values assigned to top level names in the executed code can be - retrieved by passing an empty dictionary as the first entry in the chain. If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module From 5adf78f546a5dc3f5b8eeaa209a2e8437ae96ac8 Mon Sep 17 00:00:00 2001 From: Mathijs Mortimer Date: Wed, 22 May 2024 10:51:25 +0200 Subject: [PATCH 169/903] Clarify that dklen is expected in bytes for the hashlib functions (GH-106624) --- Doc/library/hashlib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 8cf413fbd1e005..5d24b77e13bfce 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -328,7 +328,7 @@ include a `salt `_. your application, read *Appendix A.2.2* of NIST-SP-800-132_. The answers on the `stackexchange pbkdf2 iterations question`_ explain in detail. - *dklen* is the length of the derived key. If *dklen* is ``None`` then the + *dklen* is the length of the derived key in bytes. If *dklen* is ``None`` then the digest size of the hash algorithm *hash_name* is used, e.g. 64 for SHA-512. >>> from hashlib import pbkdf2_hmac @@ -357,7 +357,7 @@ include a `salt `_. *n* is the CPU/Memory cost factor, *r* the block size, *p* parallelization factor and *maxmem* limits memory (OpenSSL 1.1.0 defaults to 32 MiB). - *dklen* is the length of the derived key. + *dklen* is the length of the derived key in bytes. .. versionadded:: 3.6 From 904e256292a0ed6c36f3490da9e4f8245370f535 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Wed, 22 May 2024 17:31:37 +0800 Subject: [PATCH 170/903] Fix typos in NEWS entries for 3.13 (GH-119374) --- Misc/NEWS.d/3.13.0a1.rst | 32 ++++++++++++++++---------------- Misc/NEWS.d/3.13.0b1.rst | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 0092db29460c37..0ff50e07606aa4 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -563,7 +563,7 @@ on deallocation. Fix :meth:`multiprocessing.synchronize.SemLock.__setstate__` to properly initialize :attr:`multiprocessing.synchronize.SemLock._is_fork_ctx`. This -fixes a regression when passing a SemLock accross nested processes. +fixes a regression when passing a SemLock across nested processes. Rename :attr:`multiprocessing.synchronize.SemLock.is_fork_ctx` to :attr:`multiprocessing.synchronize.SemLock._is_fork_ctx` to avoid exposing @@ -708,7 +708,7 @@ Fixes crash when tracing in recursive calls to Python classes. .. section: Core and Builtins Remove the ``_PyCFrame`` struct, moving the pointer to the current -intepreter frame back to the threadstate, as it was for 3.10 and earlier. +interpreter frame back to the threadstate, as it was for 3.10 and earlier. The ``_PyCFrame`` existed as a performance optimization for tracing. Since PEP 669 has been implemented, this optimization no longer applies. @@ -926,7 +926,7 @@ Isolate :mod:`!_decimal` (apply :pep:`687`). Patch by Charlie Zhao. Add the exception as the third argument to ``PY_UNIND`` callbacks in ``sys.monitoring``. This makes the ``PY_UNWIND`` callback consistent with -the other exception hanlding callbacks. +the other exception handling callbacks. .. @@ -935,7 +935,7 @@ the other exception hanlding callbacks. .. nonce: DdEwV8 .. section: Core and Builtins -Raise a ``ValueError`` when a monitoring callback funtion returns +Raise a ``ValueError`` when a monitoring callback function returns ``DISABLE`` for events that cannot be disabled locally. .. @@ -1006,7 +1006,7 @@ Add :meth:`dbm.gnu.gdbm.clear` to :mod:`dbm.gnu`. Patch By Donghee Na. .. section: Core and Builtins The ASYNC and AWAIT tokens are removed from the Grammar, which removes the -posibility of making ``async`` and ``await`` soft keywords when using +possibility of making ``async`` and ``await`` soft keywords when using ``feature_version<7`` in :func:`ast.parse`. .. @@ -1028,7 +1028,7 @@ the call is not a classmethod. .. nonce: DdqHFg .. section: Core and Builtins -Python no longer crashes due an infrequent race when initialzing +Python no longer crashes due an infrequent race when initializing per-interpreter interned strings. The crash would manifest when the interpreter was finalized. @@ -1922,7 +1922,7 @@ objects .. nonce: RDGe8- .. section: Library -Deprecation warning about non-integer number in :mod:`gettext` now alwais +Deprecation warning about non-integer number in :mod:`gettext` now always refers to the line in the user code where gettext function or method is used. Previously it could refer to a line in ``gettext`` code. @@ -2047,7 +2047,7 @@ point. .. nonce: fECxTj .. section: Library -On Windows, multiprocessing ``Popen.terminate()`` now catchs +On Windows, multiprocessing ``Popen.terminate()`` now catches :exc:`PermissionError` and get the process exit code. If the process is still running, raise again the :exc:`PermissionError`. Otherwise, the process terminated as expected: store its exit code. Patch by Victor @@ -2857,7 +2857,7 @@ Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` instead of ``SSL_ERROR_SSL`` when a certification verification has failed, but the error parameters will still contain ``ERR_LIB_SSL`` and ``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and -raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +raising the appropriate ``ssl.SSLCertVerificationError``. Patch by Pablo Galindo .. @@ -2979,7 +2979,7 @@ method. Patch by James Cave. .. section: Library Fix overflow on 32-bit systems with :mod:`asyncio` :func:`os.sendfile` -implemention. +implementation. .. @@ -3251,7 +3251,7 @@ Eliseev. .. nonce: NN35-U .. section: Library -Optimize ``(?!)`` (pattern which alwais fails) in regular expressions. +Optimize ``(?!)`` (pattern which always fails) in regular expressions. .. @@ -3495,7 +3495,7 @@ star imports. .. nonce: TJEUkd .. section: Library -Zipapp will now skip over apending an archive to itself. +Zipapp will now skip over appending an archive to itself. .. @@ -4564,7 +4564,7 @@ Deprecate passing any arguments to :func:`threading.RLock`. .. nonce: o5Zb0t .. section: Library -Refactored ``zipfile._strip_extra`` to use higher level abstactions for +Refactored ``zipfile._strip_extra`` to use higher level abstractions for extras instead of a heavy-state loop. .. @@ -5014,7 +5014,7 @@ by Victor Stinner. Fix test_timeout() of test_concurrent_futures.test_wait. Remove the future which may or may not complete depending if it takes longer than the timeout -ot not. Keep the second future which does not complete before wait() +or not. Keep the second future which does not complete before wait() timeout. Patch by Victor Stinner. .. @@ -5104,7 +5104,7 @@ Victor Stinner. regrtest: Add ``--fast-ci`` and ``--slow-ci`` options. ``--fast-ci`` uses a default timeout of 10 minutes and ``-u all,-cpu`` (skip slowest tests). -``--slow-ci`` uses a default timeout of 20 minues and ``-u all`` (run all +``--slow-ci`` uses a default timeout of 20 minutes and ``-u all`` (run all tests). Patch by Victor Stinner. .. @@ -5232,7 +5232,7 @@ and ``sysctl net.inet.udp.blackhole=1``). Patch by Victor Stinner. Skip ``test_gdb`` if gdb is unable to retrieve Python frame objects: if a frame is ````. When Python is built with "clang -Og", gdb can -fail to retrive the *frame* parameter of ``_PyEval_EvalFrameDefault()``. In +fail to retrieve the *frame* parameter of ``_PyEval_EvalFrameDefault()``. In this case, tests like ``py_bt()`` are likely to fail. Without getting access to Python frames, ``python-gdb.py`` is mostly clueless on retrieving the Python traceback. Moreover, ``test_gdb`` is no longer skipped on macOS if diff --git a/Misc/NEWS.d/3.13.0b1.rst b/Misc/NEWS.d/3.13.0b1.rst index c3bfcc83f2ae7c..525491a2603416 100644 --- a/Misc/NEWS.d/3.13.0b1.rst +++ b/Misc/NEWS.d/3.13.0b1.rst @@ -1351,7 +1351,7 @@ before creating the :class:`~tkinter.Tk` object or call the :meth:`~tkinter.Tk.wantobject` method of the :class:`!Tk` object with argument ``2`` makes now arguments to callbacks registered in the :mod:`tkinter` module to be passed as various Python objects (``int``, ``float``, ``bytes``, ``tuple``), -depending on their internal represenation in Tcl, instead of always ``str``. +depending on their internal representation in Tcl, instead of always ``str``. :data:`!tkinter.wantobject` is now set to ``2`` by default. .. From aee8f03abbebfb76357f459dfb297026862e3c0b Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 22 May 2024 05:43:56 -0400 Subject: [PATCH 171/903] Fix version number in use_load_tests deprecation reference (GH-119151) Deprecation took place in d78742a260ba09e53c844de7b1fd11a11c674945 (3.5) --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 685d0120f2f636..b64e4e205fe8c1 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1739,7 +1739,7 @@ unittest * Undocumented :meth:`TestLoader.loadTestsFromModule ` parameter *use_load_tests* - (deprecated and ignored since Python 3.2). + (deprecated and ignored since Python 3.5). * An alias of the :class:`~unittest.TextTestResult` class: ``_TextTestResult`` (deprecated in Python 3.2). From 858b9e85fcdd495947c9e892ce6e3734652c48f2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 22 May 2024 13:17:46 +0300 Subject: [PATCH 172/903] gh-118643: Fix AttributeError in the email module (GH-119099) Fix regression introduced in gh-100884: AttributeError when re-fold a long address list. Also fix more cases of incorrect encoding of the address separator in the address list missed in gh-100884. --- Lib/email/_header_value_parser.py | 15 ++++++++++++--- Lib/test/test_email/test__header_value_parser.py | 12 ++++++++++-- ...2024-05-16-17-31-46.gh-issue-118643.hAWH4C.rst | 2 ++ 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-16-17-31-46.gh-issue-118643.hAWH4C.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 6148801460cfb2..ab3c3031ef590c 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -956,6 +956,7 @@ class _InvalidEwError(errors.HeaderParseError): DOT = ValueTerminal('.', 'dot') ListSeparator = ValueTerminal(',', 'list-separator') ListSeparator.as_ew_allowed = False +ListSeparator.syntactic_break = False RouteComponentMarker = ValueTerminal('@', 'route-component-marker') # @@ -2844,7 +2845,9 @@ def _refold_parse_tree(parse_tree, *, policy): if not hasattr(part, 'encode'): # It's not a Terminal, do each piece individually. parts = list(part) + parts - else: + want_encoding = False + continue + elif part.as_ew_allowed: # It's a terminal, wrap it as an encoded word, possibly # combining it with previously encoded words if allowed. if (last_ew is not None and @@ -2858,8 +2861,14 @@ def _refold_parse_tree(parse_tree, *, policy): # so clear it now. leading_whitespace = '' last_charset = charset - want_encoding = False - continue + want_encoding = False + continue + else: + # It's a terminal which should be kept non-encoded + # (e.g. a ListSeparator). + last_ew = None + want_encoding = False + # fall through if len(tstr) <= maxlen - len(lines[-1]): lines[-1] += tstr diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 56a1e3a3de5aa2..5413319a414a62 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -3077,9 +3077,17 @@ def test_address_list_with_unicode_names_in_quotes(self): ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= \n') def test_address_list_with_list_separator_after_fold(self): - to = '0123456789' * 8 + '@foo, ä ' + a = 'x' * 66 + '@example.com' + to = f'{a}, "Hübsch Kaktus" ' self._test(parser.get_address_list(to)[0], - '0123456789' * 8 + '@foo,\n =?utf-8?q?=C3=A4?= \n') + f'{a},\n =?utf-8?q?H=C3=BCbsch?= Kaktus \n') + + a = '.' * 79 + to = f'"{a}" , "Hübsch Kaktus" ' + self._test(parser.get_address_list(to)[0], + f'{a}\n' + ' , =?utf-8?q?H=C3=BCbsch?= Kaktus ' + '\n') # XXX Need tests with comments on various sides of a unicode token, # and with unicode tokens in the comments. Spaces inside the quotes diff --git a/Misc/NEWS.d/next/Library/2024-05-16-17-31-46.gh-issue-118643.hAWH4C.rst b/Misc/NEWS.d/next/Library/2024-05-16-17-31-46.gh-issue-118643.hAWH4C.rst new file mode 100644 index 00000000000000..e86a49af74c9d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-16-17-31-46.gh-issue-118643.hAWH4C.rst @@ -0,0 +1,2 @@ +Fix an AttributeError in the :mod:`email` module when re-fold a long address +list. Also fix more cases of incorrect encoding of the address separator in the address list. From d472b4f9fa4fb6061588d421f33a0388a2005bc6 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Wed, 22 May 2024 14:52:36 +0200 Subject: [PATCH 173/903] gh-119391: Amend comment description of PyMapping_Items, PyMapping_Values and PyMapping_Keys (#119392) The behaviour was changed in 0ccc0f6c7. --- Include/abstract.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/abstract.h b/Include/abstract.h index bd12a54963c13f..f0e49c1afb8164 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -852,15 +852,15 @@ PyAPI_FUNC(int) PyMapping_HasKeyWithError(PyObject *o, PyObject *key); PyAPI_FUNC(int) PyMapping_HasKeyStringWithError(PyObject *o, const char *key); -/* On success, return a list or tuple of the keys in mapping object 'o'. +/* On success, return a list of the keys in mapping object 'o'. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Keys(PyObject *o); -/* On success, return a list or tuple of the values in mapping object 'o'. +/* On success, return a list of the values in mapping object 'o'. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Values(PyObject *o); -/* On success, return a list or tuple of the items in mapping object 'o', +/* On success, return a list of the items in mapping object 'o', where each item is a tuple containing a key-value pair. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Items(PyObject *o); From 81865002aee8eaaeb3c7e402f86183afa6de77bf Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 22 May 2024 11:57:52 -0400 Subject: [PATCH 174/903] gh-119213: Be More Careful About _PyArg_Parser.kwtuple Across Interpreters (gh-119331) _PyArg_Parser holds static global data generated for modules by Argument Clinic. The _PyArg_Parser.kwtuple field is a tuple object, even though it's stored within a static global. In some cases the tuple is statically allocated and thus it's okay that it gets shared by multiple interpreters. However, in other cases the tuple is set lazily, allocated from the heap using the active interprepreter at the point the tuple is needed. This is a problem once that interpreter is destroyed since _PyArg_Parser.kwtuple becomes at dangling pointer, leading to crashes. It isn't a problem if the tuple is allocated under the main interpreter, since its lifetime is bound to the lifetime of the runtime. The solution here is to temporarily switch to the main interpreter. The alternative would be to always statically allocate the tuple. This change also fixes a bug where only the most recent parser was added to the global linked list. --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/test_capi/test_getargs.py | 33 ++++++++++ ...-05-21-11-27-14.gh-issue-119213.nxjxrt.rst | 3 + Modules/_testinternalcapi.c | 20 ++++++ Modules/clinic/_testinternalcapi.c.h | 62 ++++++++++++++++++- Python/getargs.c | 21 ++++++- Tools/clinic/libclinic/parse_args.py | 2 + 10 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-21-11-27-14.gh-issue-119213.nxjxrt.rst diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 33133aaaf00893..56a2d6b0f4fc5d 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1222,6 +1222,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sort)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(source)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(source_traceback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(spam)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src_dir_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stacklevel)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index f5ea7b9bd7d433..657eac6c0a0f6c 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -711,6 +711,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(sort) STRUCT_FOR_ID(source) STRUCT_FOR_ID(source_traceback) + STRUCT_FOR_ID(spam) STRUCT_FOR_ID(src) STRUCT_FOR_ID(src_dir_fd) STRUCT_FOR_ID(stacklevel) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index c73408d6315312..f4f9c730e514e0 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1220,6 +1220,7 @@ extern "C" { INIT_ID(sort), \ INIT_ID(source), \ INIT_ID(source_traceback), \ + INIT_ID(spam), \ INIT_ID(src), \ INIT_ID(src_dir_fd), \ INIT_ID(stacklevel), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index d84c45a6b57887..33da27a941f024 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1974,6 +1974,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(source_traceback); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(spam); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(src); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index e710400f75c235..232aa2a80025dc 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -4,11 +4,17 @@ import sys from test import support from test.support import import_helper +from test.support import script_helper from test.support import warnings_helper # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') from _testcapi import getargs_keywords, getargs_keyword_only +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = NULL + # > How about the following counterproposal. This also changes some of # > the other format codes to be a little more regular. # > @@ -1346,6 +1352,33 @@ def test_nested_tuple(self): "argument 1 must be sequence of length 1, not 0"): parse(((),), {}, '(' + f + ')', ['a']) + @unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi') + def test_gh_119213(self): + rc, out, err = script_helper.assert_python_ok("-c", """if True: + from test import support + script = '''if True: + import _testinternalcapi + _testinternalcapi.gh_119213_getargs(spam='eggs') + ''' + config = dict( + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + use_main_obmalloc=False, + gil=2, + check_multi_interp_extensions=True, + ) + rc = support.run_in_subinterp_with_config(script, **config) + assert rc == 0 + + # The crash is different if the interpreter was not destroyed first. + #interpid = _testinternalcapi.create_interpreter() + #rc = _testinternalcapi.exec_interpreter(interpid, script) + #assert rc == 0 + """) + self.assertEqual(rc, 0) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-21-11-27-14.gh-issue-119213.nxjxrt.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-11-27-14.gh-issue-119213.nxjxrt.rst new file mode 100644 index 00000000000000..e9073b4ba08798 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-11-27-14.gh-issue-119213.nxjxrt.rst @@ -0,0 +1,3 @@ +Non-builtin modules built with argument clinic were crashing if used in a +subinterpreter before the main interpreter. The objects that were causing +the problem by leaking between interpreters carelessly have been fixed. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index e209c7e05264f2..129c136906739d 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2006,6 +2006,25 @@ has_inline_values(PyObject *self, PyObject *obj) Py_RETURN_FALSE; } + +/*[clinic input] +gh_119213_getargs + + spam: object = None + +Test _PyArg_Parser.kwtuple +[clinic start generated code]*/ + +static PyObject * +gh_119213_getargs_impl(PyObject *module, PyObject *spam) +/*[clinic end generated code: output=d8d9c95d5b446802 input=65ef47511da80fc2]*/ +{ + // It must never have been called in the main interprer + assert(!_Py_IsMainInterpreter(PyInterpreterState_Get())); + return Py_NewRef(spam); +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2096,6 +2115,7 @@ static PyMethodDef module_functions[] = { #ifdef _Py_TIER2 {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, #endif + GH_119213_GETARGS_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index a61858561d5ef8..dcca9ecf735723 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -300,4 +300,64 @@ _testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignore { return _testinternalcapi_test_long_numbits_impl(module); } -/*[clinic end generated code: output=9804015d77d36c77 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(gh_119213_getargs__doc__, +"gh_119213_getargs($module, /, spam=None)\n" +"--\n" +"\n" +"Test _PyArg_Parser.kwtuple"); + +#define GH_119213_GETARGS_METHODDEF \ + {"gh_119213_getargs", _PyCFunction_CAST(gh_119213_getargs), METH_FASTCALL|METH_KEYWORDS, gh_119213_getargs__doc__}, + +static PyObject * +gh_119213_getargs_impl(PyObject *module, PyObject *spam); + +static PyObject * +gh_119213_getargs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(spam), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"spam", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "gh_119213_getargs", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *spam = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + spam = args[0]; +skip_optional_pos: + return_value = gh_119213_getargs_impl(module, spam); + +exit: + return return_value; +} +/*[clinic end generated code: output=4d0770a1c20fbf40 input=a9049054013a1b77]*/ diff --git a/Python/getargs.c b/Python/getargs.c index 539925e471f54c..f9a836679fe55e 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -7,6 +7,7 @@ #include "pycore_dict.h" // _PyDict_HasOnlyStringKeys() #include "pycore_modsupport.h" // export _PyArg_NoKeywords() #include "pycore_pylifecycle.h" // _PyArg_Fini +#include "pycore_pystate.h" // _Py_IsMainInterpreter() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_pyerrors.h" // _Py_CalculateSuggestions() @@ -1947,7 +1948,23 @@ _parser_init(void *arg) int owned; PyObject *kwtuple = parser->kwtuple; if (kwtuple == NULL) { + /* We may temporarily switch to the main interpreter to avoid + * creating a tuple that could outlive its owning interpreter. */ + PyThreadState *save_tstate = NULL; + PyThreadState *temp_tstate = NULL; + if (!_Py_IsMainInterpreter(PyInterpreterState_Get())) { + temp_tstate = PyThreadState_New(_PyInterpreterState_Main()); + if (temp_tstate == NULL) { + return -1; + } + save_tstate = PyThreadState_Swap(temp_tstate); + } kwtuple = new_kwtuple(keywords, len, pos); + if (temp_tstate != NULL) { + PyThreadState_Clear(temp_tstate); + (void)PyThreadState_Swap(save_tstate); + PyThreadState_Delete(temp_tstate); + } if (kwtuple == NULL) { return -1; } @@ -1969,8 +1986,8 @@ _parser_init(void *arg) parser->next = _Py_atomic_load_ptr(&_PyRuntime.getargs.static_parsers); do { // compare-exchange updates parser->next on failure - } while (_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers, - &parser->next, parser)); + } while (!_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers, + &parser->next, parser)); return 0; } diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index 905f2a0ba94f4c..7ac88bd0458b82 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -51,6 +51,8 @@ def declare_parser( #endif """ else: + # XXX Why do we not statically allocate the tuple + # for non-builtin modules? declarations = """ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) From ef172521a9e9dfadebe57d590bfb53a0e9ac3a0b Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Wed, 22 May 2024 12:35:18 -0400 Subject: [PATCH 175/903] Remove almost all unpaired backticks in docstrings (#119231) As reported in #117847 and #115366, an unpaired backtick in a docstring tends to confuse e.g. Sphinx running on subclasses of standard library objects, and the typographic style of using a backtick as an opening quote is no longer in favor. Convert almost all uses of the form The variable `foo' should do xyz to The variable 'foo' should do xyz and also fix up miscellaneous other unpaired backticks (extraneous / missing characters). No functional change is intended here other than in human-readable docstrings. --- Lib/_pyrepl/keymap.py | 18 +++---- Lib/_pyrepl/reader.py | 8 +-- Lib/cmd.py | 24 ++++----- Lib/configparser.py | 2 +- Lib/doctest.py | 4 +- Lib/email/_parseaddr.py | 14 ++--- Lib/email/_policybase.py | 2 +- Lib/email/base64mime.py | 2 +- Lib/email/charset.py | 4 +- Lib/email/generator.py | 6 +-- Lib/email/header.py | 6 +-- Lib/email/iterators.py | 4 +- Lib/email/message.py | 26 ++++----- Lib/email/mime/multipart.py | 2 +- Lib/email/parser.py | 4 +- Lib/email/quoprimime.py | 8 +-- Lib/ftplib.py | 2 +- Lib/getopt.py | 6 +-- Lib/heapq.py | 6 +-- Lib/http/client.py | 10 ++-- Lib/http/cookiejar.py | 2 +- Lib/imaplib.py | 10 ++-- Lib/mimetypes.py | 26 ++++----- Lib/smtplib.py | 6 +-- Lib/tarfile.py | 60 ++++++++++----------- Lib/test/support/smtpd.py | 14 ++--- Lib/test/test_asyncio/test_locks.py | 4 +- Lib/trace.py | 4 +- Lib/unittest/mock.py | 2 +- Lib/wsgiref/headers.py | 2 +- Lib/zipfile/__init__.py | 8 +-- Modules/_heapqmodule.c | 6 +-- Modules/_interpretersmodule.c | 2 +- Modules/cjkcodecs/clinic/multibytecodec.c.h | 4 +- Modules/cjkcodecs/multibytecodec.c | 4 +- Modules/clinic/pyexpat.c.h | 4 +- Modules/pyexpat.c | 4 +- Objects/bytesobject.c | 22 ++++---- Objects/odictobject.c | 2 +- 39 files changed, 172 insertions(+), 172 deletions(-) diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py index 2fb03d1952382f..d11df4b5164696 100644 --- a/Lib/_pyrepl/keymap.py +++ b/Lib/_pyrepl/keymap.py @@ -30,20 +30,20 @@ pyrepl uses its own keyspec format that is meant to be a strict superset of readline's KEYSEQ format. This means that if a spec is found that readline accepts that this doesn't, it should be logged as a bug. Note that this means -we're using the `\\C-o' style of readline's keyspec, not the `Control-o' sort. +we're using the '\\C-o' style of readline's keyspec, not the 'Control-o' sort. The extension to readline is that the sequence \\ denotes the sequence of characters produced by hitting KEY. Examples: -`a' - what you get when you hit the `a' key -`\\EOA' - Escape - O - A (up, on my terminal) -`\\' - the up arrow key -`\\' - ditto (keynames are case-insensitive) -`\\C-o', `\\c-o' - control-o -`\\M-.' - meta-period -`\\E.' - ditto (that's how meta works for pyrepl) -`\\', `\\', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' +'a' - what you get when you hit the 'a' key +'\\EOA' - Escape - O - A (up, on my terminal) +'\\' - the up arrow key +'\\' - ditto (keynames are case-insensitive) +'\\C-o', '\\c-o' - control-o +'\\M-.' - meta-period +'\\E.' - ditto (that's how meta works for pyrepl) +'\\', '\\', '\\t', '\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' - all of these are the tab character. """ diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 3dbfc34780e8e8..768d45a045e3be 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -171,7 +171,7 @@ class Reader: * console: Hopefully encapsulates the OS dependent stuff. * pos: - A 0-based index into `buffer' for where the insertion point + A 0-based index into 'buffer' for where the insertion point is. * screeninfo: Ahem. This list contains some info needed to move the @@ -179,7 +179,7 @@ class Reader: * cxy, lxy: the position of the insertion point in screen ... * syntax_table: - Dictionary mapping characters to `syntax class'; read the + Dictionary mapping characters to 'syntax class'; read the emacs docs to see what this means :-) * commands: Dictionary mapping command names to command classes. @@ -431,7 +431,7 @@ def max_row(self) -> int: def get_arg(self, default: int = 1) -> int: """Return any prefix argument that the user has supplied, - returning `default' if there is None. Defaults to 1. + returning 'default' if there is None. Defaults to 1. """ if self.arg is None: return default @@ -440,7 +440,7 @@ def get_arg(self, default: int = 1) -> int: def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: """Return what should be in the left-hand margin for line - `lineno'.""" + 'lineno'.""" if self.arg is not None and cursor_on_line: prompt = "(arg: %s) " % self.arg elif self.paste_mode: diff --git a/Lib/cmd.py b/Lib/cmd.py index a37d16cd7bde16..c333e099bd8c9a 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -5,16 +5,16 @@ 1. End of file on input is processed as the command 'EOF'. 2. A command is parsed out of each line by collecting the prefix composed of characters in the identchars member. -3. A command `foo' is dispatched to a method 'do_foo()'; the do_ method +3. A command 'foo' is dispatched to a method 'do_foo()'; the do_ method is passed a single argument consisting of the remainder of the line. 4. Typing an empty line repeats the last command. (Actually, it calls the - method `emptyline', which may be overridden in a subclass.) -5. There is a predefined `help' method. Given an argument `topic', it - calls the command `help_topic'. With no arguments, it lists all topics + method 'emptyline', which may be overridden in a subclass.) +5. There is a predefined 'help' method. Given an argument 'topic', it + calls the command 'help_topic'. With no arguments, it lists all topics with defined help_ functions, broken into up to three topics; documented commands, miscellaneous help topics, and undocumented commands. -6. The command '?' is a synonym for `help'. The command '!' is a synonym - for `shell', if a do_shell method exists. +6. The command '?' is a synonym for 'help'. The command '!' is a synonym + for 'shell', if a do_shell method exists. 7. If completion is enabled, completing commands will be done automatically, and completing of commands args is done by calling complete_foo() with arguments text, line, begidx, endidx. text is string we are matching @@ -23,21 +23,21 @@ indexes of the text being matched, which could be used to provide different completion depending upon which position the argument is in. -The `default' method may be overridden to intercept commands for which there +The 'default' method may be overridden to intercept commands for which there is no do_ method. -The `completedefault' method may be overridden to intercept completions for +The 'completedefault' method may be overridden to intercept completions for commands that have no complete_ method. -The data member `self.ruler' sets the character used to draw separator lines +The data member 'self.ruler' sets the character used to draw separator lines in the help messages. If empty, no ruler line is drawn. It defaults to "=". -If the value of `self.intro' is nonempty when the cmdloop method is called, +If the value of 'self.intro' is nonempty when the cmdloop method is called, it is printed out on interpreter startup. This value may be overridden via an optional argument to the cmdloop() method. -The data members `self.doc_header', `self.misc_header', and -`self.undoc_header' set the headers used for the help function's +The data members 'self.doc_header', 'self.misc_header', and +'self.undoc_header' set the headers used for the help function's listings of documented functions, miscellaneous topics, and undocumented functions respectively. """ diff --git a/Lib/configparser.py b/Lib/configparser.py index ff7d712bed4afc..4344a9e8baca44 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -957,7 +957,7 @@ def write(self, fp, space_around_delimiters=True): self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False): - """Write a single section to the specified `fp'.""" + """Write a single section to the specified 'fp'.""" if not unnamed: fp.write("[{}]\n".format(section_name)) for key, value in section_items: diff --git a/Lib/doctest.py b/Lib/doctest.py index c531e3ca6a3d5e..ea7d275c91db04 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1227,7 +1227,7 @@ class DocTestRunner: `OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. - First, an output function (`out) can be passed to + First, an output function (`out`) can be passed to `TestRunner.run`; this function will be called with strings that should be displayed. It defaults to `sys.stdout.write`. If capturing the output is not sufficient, then the display output @@ -2734,7 +2734,7 @@ def testsource(module, name): return testsrc def debug_src(src, pm=False, globs=None): - """Debug a single doctest docstring, in argument `src`'""" + """Debug a single doctest docstring, in argument `src`""" testsrc = script_from_examples(src) debug_script(testsrc, pm, globs) diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 0f1bf8e4253ec4..36625e35ffb6a7 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -224,7 +224,7 @@ class AddrlistClass: def __init__(self, field): """Initialize a new instance. - `field' is an unparsed address header field, containing + 'field' is an unparsed address header field, containing one or more addresses. """ self.specials = '()<>@,:;.\"[]' @@ -233,7 +233,7 @@ def __init__(self, field): self.CR = '\r\n' self.FWS = self.LWS + self.CR self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it + # Note that RFC 2822 now specifies '.' as obs-phrase, meaning that it # is obsolete syntax. RFC 2822 requires that we recognize obsolete # syntax, so allow dots in phrases. self.phraseends = self.atomends.replace('.', '') @@ -423,14 +423,14 @@ def getdomain(self): def getdelimited(self, beginchar, endchars, allowcomments=True): """Parse a header fragment delimited by special characters. - `beginchar' is the start character for the fragment. - If self is not looking at an instance of `beginchar' then + 'beginchar' is the start character for the fragment. + If self is not looking at an instance of 'beginchar' then getdelimited returns the empty string. - `endchars' is a sequence of allowable end-delimiting characters. + 'endchars' is a sequence of allowable end-delimiting characters. Parsing stops when one of these is encountered. - If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed + If 'allowcomments' is non-zero, embedded RFC 2822 comments are allowed within the parsed fragment. """ if self.field[self.pos] != beginchar: @@ -474,7 +474,7 @@ def getatom(self, atomends=None): Optional atomends specifies a different set of end token delimiters (the default is to use self.atomends). This is used e.g. in - getphraselist() since phrase endings must not include the `.' (which + getphraselist() since phrase endings must not include the '.' (which is legal in phrases).""" atomlist = [''] if atomends is None: diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index 2ec54fbabae83c..1c76ed63b61ae8 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -150,7 +150,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): wrapping is done. Default is 78. mangle_from_ -- a flag that, when True escapes From_ lines in the - body of the message by putting a `>' in front of + body of the message by putting a '>' in front of them. This is used when the message is being serialized by a generator. Default: False. diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index 4cdf22666e3016..d440de95255bf1 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -15,7 +15,7 @@ with Base64 encoding. RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names +'encoded-word' in a header. This method is commonly used for 8-bit real names in To:, From:, Cc:, etc. fields, as well as Subject: lines. This module does not do the line wrapping or end-of-line character conversion diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 043801107b60e5..cfd5a0c456e497 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -175,7 +175,7 @@ class Charset: module expose the following information about a character set: input_charset: The initial character set specified. Common aliases - are converted to their `official' email names (e.g. latin_1 + are converted to their 'official' email names (e.g. latin_1 is converted to iso-8859-1). Defaults to 7-bit us-ascii. header_encoding: If the character set must be encoded before it can be @@ -245,7 +245,7 @@ def __eq__(self, other): def get_body_encoding(self): """Return the content-transfer-encoding used for body encoding. - This is either the string `quoted-printable' or `base64' depending on + This is either the string 'quoted-printable' or 'base64' depending on the encoding used, or it is a function in which case you should call the function with a single argument, the Message object being encoded. The function should then set the Content-Transfer-Encoding diff --git a/Lib/email/generator.py b/Lib/email/generator.py index c8056ad47baa0f..9d058ceada24f8 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -41,7 +41,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, Optional mangle_from_ is a flag that, when True (the default if policy is not set), escapes From_ lines in the body of the message by putting - a `>' in front of them. + a '>' in front of them. Optional maxheaderlen specifies the longest length for a non-continued header. When a header line is longer (in characters, with tabs @@ -74,7 +74,7 @@ def flatten(self, msg, unixfrom=False, linesep=None): unixfrom is a flag that forces the printing of a Unix From_ delimiter before the first object in the message tree. If the original message - has no From_ delimiter, a `standard' one is crafted. By default, this + has no From_ delimiter, a 'standard' one is crafted. By default, this is False to inhibit the printing of any From_ delimiter. Note that for subobjects, no From_ line is printed. @@ -456,7 +456,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, fmt=None, *, argument is allowed. Walks through all subparts of a message. If the subpart is of main - type `text', then it prints the decoded payload of the subpart. + type 'text', then it prints the decoded payload of the subpart. Otherwise, fmt is a format string that is used instead of the message payload. fmt is expanded with the following keywords (in diff --git a/Lib/email/header.py b/Lib/email/header.py index 984851a7d9a679..66a1d46db50c45 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -192,7 +192,7 @@ def __init__(self, s=None, charset=None, The maximum line length can be specified explicitly via maxlinelen. For splitting the first line to a shorter value (to account for the field - header which isn't included in s, e.g. `Subject') pass in the name of + header which isn't included in s, e.g. 'Subject') pass in the name of the field in header_name. The default maxlinelen is 78 as recommended by RFC 2822. @@ -276,7 +276,7 @@ def append(self, s, charset=None, errors='strict'): output codec of the charset. If the string cannot be encoded to the output codec, a UnicodeError will be raised. - Optional `errors' is passed as the errors argument to the decode + Optional 'errors' is passed as the errors argument to the decode call if s is a byte string. """ if charset is None: @@ -326,7 +326,7 @@ def encode(self, splitchars=';, \t', maxlinelen=None, linesep='\n'): Optional splitchars is a string containing characters which should be given extra weight by the splitting algorithm during normal header - wrapping. This is in very rough support of RFC 2822's `higher level + wrapping. This is in very rough support of RFC 2822's 'higher level syntactic breaks': split points preceded by a splitchar are preferred during line splitting, with the characters preferred in the order in which they appear in the string. Space and tab may be included in the diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index 3410935e38f476..2f436aefc2300b 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -43,8 +43,8 @@ def body_line_iterator(msg, decode=False): def typed_subpart_iterator(msg, maintype='text', subtype=None): """Iterate over the subparts with a given MIME type. - Use `maintype' as the main MIME type to match against; this defaults to - "text". Optional `subtype' is the MIME subtype to match against; if + Use 'maintype' as the main MIME type to match against; this defaults to + "text". Optional 'subtype' is the MIME subtype to match against; if omitted, only the main type is matched. """ for subpart in msg.walk(): diff --git a/Lib/email/message.py b/Lib/email/message.py index 46bb8c21942af8..08192c50a8ff5c 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -21,7 +21,7 @@ SEMISPACE = '; ' -# Regular expression that matches `special' characters in parameters, the +# Regular expression that matches 'special' characters in parameters, the # existence of which force quoting of the parameter value. tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') @@ -141,7 +141,7 @@ class Message: multipart or a message/rfc822), then the payload is a list of Message objects, otherwise it is a string. - Message objects implement part of the `mapping' interface, which assumes + Message objects implement part of the 'mapping' interface, which assumes there is exactly one occurrence of the header per message. Some headers do in fact appear multiple times (e.g. Received) and for those headers, you must use the explicit API to set or get all the headers. Not all of @@ -597,7 +597,7 @@ def get_content_type(self): """Return the message's content type. The returned string is coerced to lower case of the form - `maintype/subtype'. If there was no Content-Type header in the + 'maintype/subtype'. If there was no Content-Type header in the message, the default type as given by get_default_type() will be returned. Since according to RFC 2045, messages always have a default type this will always return a value. @@ -620,7 +620,7 @@ def get_content_type(self): def get_content_maintype(self): """Return the message's main content type. - This is the `maintype' part of the string returned by + This is the 'maintype' part of the string returned by get_content_type(). """ ctype = self.get_content_type() @@ -629,14 +629,14 @@ def get_content_maintype(self): def get_content_subtype(self): """Returns the message's sub-content type. - This is the `subtype' part of the string returned by + This is the 'subtype' part of the string returned by get_content_type(). """ ctype = self.get_content_type() return ctype.split('/')[1] def get_default_type(self): - """Return the `default' content type. + """Return the 'default' content type. Most messages have a default content type of text/plain, except for messages that are subparts of multipart/digest containers. Such @@ -645,7 +645,7 @@ def get_default_type(self): return self._default_type def set_default_type(self, ctype): - """Set the `default' content type. + """Set the 'default' content type. ctype should be either "text/plain" or "message/rfc822", although this is not enforced. The default content type is not stored in the @@ -678,8 +678,8 @@ def get_params(self, failobj=None, header='content-type', unquote=True): """Return the message's Content-Type parameters, as a list. The elements of the returned list are 2-tuples of key/value pairs, as - split on the `=' sign. The left hand side of the `=' is the key, - while the right hand side is the value. If there is no `=' sign in + split on the '=' sign. The left hand side of the '=' is the key, + while the right hand side is the value. If there is no '=' sign in the parameter the value is the empty string. The value is as described in the get_param() method. @@ -839,9 +839,9 @@ def get_filename(self, failobj=None): """Return the filename associated with the payload if present. The filename is extracted from the Content-Disposition header's - `filename' parameter, and it is unquoted. If that header is missing - the `filename' parameter, this method falls back to looking for the - `name' parameter. + 'filename' parameter, and it is unquoted. If that header is missing + the 'filename' parameter, this method falls back to looking for the + 'name' parameter. """ missing = object() filename = self.get_param('filename', missing, 'content-disposition') @@ -854,7 +854,7 @@ def get_filename(self, failobj=None): def get_boundary(self, failobj=None): """Return the boundary associated with the payload if present. - The boundary is extracted from the Content-Type header's `boundary' + The boundary is extracted from the Content-Type header's 'boundary' parameter, and it is unquoted. """ missing = object() diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 94d81c771a474e..47fc218e1ae032 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -21,7 +21,7 @@ def __init__(self, _subtype='mixed', boundary=None, _subparts=None, Content-Type and MIME-Version headers. _subtype is the subtype of the multipart content type, defaulting to - `mixed'. + 'mixed'. boundary is the multipart boundary string. By default it is calculated as needed. diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 06d99b17f2f9c4..475aa2b1a66680 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -22,7 +22,7 @@ def __init__(self, _class=None, *, policy=compat32): textual representation of the message. The string must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the string or by a blank line. @@ -82,7 +82,7 @@ def __init__(self, *args, **kw): textual representation of the message. The input must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the input or by a blank line. diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py index 27fcbb5a26e3ae..500bbc5151769d 100644 --- a/Lib/email/quoprimime.py +++ b/Lib/email/quoprimime.py @@ -5,7 +5,7 @@ """Quoted-printable content transfer encoding per RFCs 2045-2047. This module handles the content transfer encoding method defined in RFC 2045 -to encode US ASCII-like 8-bit data called `quoted-printable'. It is used to +to encode US ASCII-like 8-bit data called 'quoted-printable'. It is used to safely encode text that is in a character set similar to the 7-bit US ASCII character set, but that includes some 8-bit characters that are normally not allowed in email bodies or headers. @@ -17,7 +17,7 @@ with quoted-printable encoding. RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names +'encoded-word' in a header. This method is commonly used for 8-bit real names in To:/From:/Cc: etc. fields, as well as Subject: lines. This module does not do the line wrapping or end-of-line character @@ -127,7 +127,7 @@ def quote(c): def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with quoted-printable (like) encoding. - Defined in RFC 2045, this `Q' encoding is similar to quoted-printable, but + Defined in RFC 2045, this 'Q' encoding is similar to quoted-printable, but used specifically for email header fields to allow charsets with mostly 7 bit characters (and some 8 bit) to remain more or less readable in non-RFC 2045 aware mail clients. @@ -290,7 +290,7 @@ def _unquote_match(match): # Header decoding is done a bit differently def header_decode(s): - """Decode a string encoded with RFC 2045 MIME header `Q' encoding. + """Decode a string encoded with RFC 2045 MIME header 'Q' encoding. This function does not parse a full MIME header value encoded with quoted-printable (like =?iso-8859-1?q?Hello_World?=) -- please use diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 10c5d1ea08ab11..50771e8c17c250 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -343,7 +343,7 @@ def ntransfercmd(self, cmd, rest=None): connection and the expected size of the transfer. The expected size may be None if it could not be determined. - Optional `rest' argument can be a string that is sent as the + Optional 'rest' argument can be a string that is sent as the argument to a REST command. This is essentially a server marker used to tell the server to skip over any data up to the given marker. diff --git a/Lib/getopt.py b/Lib/getopt.py index 5419d77f5d774e..e5fd04fe12a7ee 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -2,8 +2,8 @@ This module helps scripts to parse the command line arguments in sys.argv. It supports the same conventions as the Unix getopt() -function (including the special meanings of arguments of the form `-' -and `--'). Long options similar to those supported by GNU software +function (including the special meanings of arguments of the form '-' +and '--'). Long options similar to those supported by GNU software may be used as well via an optional third argument. This module provides two functions and an exception: @@ -105,7 +105,7 @@ def gnu_getopt(args, shortopts, longopts = []): processing options as soon as a non-option argument is encountered. - If the first character of the option string is `+', or if the + If the first character of the option string is '+', or if the environment variable POSIXLY_CORRECT is set, then option processing stops as soon as a non-option argument is encountered. diff --git a/Lib/heapq.py b/Lib/heapq.py index 2fd9d1ff4bf827..c53cb5537db1a4 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -42,7 +42,7 @@ property of a heap is that a[0] is always its smallest element. The strange invariant above is meant to be an efficient memory -representation for a tournament. The numbers below are `k', not a[k]: +representation for a tournament. The numbers below are 'k', not a[k]: 0 @@ -55,7 +55,7 @@ 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 -In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In +In the tree above, each cell 'k' is topping '2*k+1' and '2*k+2'. In a usual binary tournament we see in sports, each cell is the winner over the two cells it tops, and we can trace the winner down the tree to see all opponents s/he had. However, in many computer applications @@ -110,7 +110,7 @@ effective! In a word, heaps are useful memory structures to know. I use them in -a few applications, and I think it is good to keep a `heap' module +a few applications, and I think it is good to keep a 'heap' module around. :-) -------------------- diff --git a/Lib/http/client.py b/Lib/http/client.py index a353716a8506e6..fab90a0ba4eb83 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1025,7 +1025,7 @@ def close(self): response.close() def send(self, data): - """Send `data' to the server. + """Send 'data' to the server. ``data`` can be a string object, a bytes object, an array object, a file-like object that supports a .read() method, or an iterable object. """ @@ -1137,10 +1137,10 @@ def putrequest(self, method, url, skip_host=False, skip_accept_encoding=False): """Send a request to the server. - `method' specifies an HTTP request method, e.g. 'GET'. - `url' specifies the object being requested, e.g. '/index.html'. - `skip_host' if True does not add automatically a 'Host:' header - `skip_accept_encoding' if True does not add automatically an + 'method' specifies an HTTP request method, e.g. 'GET'. + 'url' specifies the object being requested, e.g. '/index.html'. + 'skip_host' if True does not add automatically a 'Host:' header + 'skip_accept_encoding' if True does not add automatically an 'Accept-Encoding:' header """ diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index bd89370e16831e..fb0fd2e97999af 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1986,7 +1986,7 @@ class MozillaCookieJar(FileCookieJar): This class differs from CookieJar only in the format it uses to save and load cookies to and from a file. This class uses the Mozilla/Netscape - `cookies.txt' format. curl and lynx use this file format, too. + 'cookies.txt' format. curl and lynx use this file format, too. Don't expect cookies saved while the browser is running to be noticed by the browser (in fact, Mozilla on unix will overwrite your saved cookies if diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 577b4b9b03a88d..e576c29e67dc0a 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -239,7 +239,7 @@ def _connect(self): if __debug__: self._cmd_log_len = 10 self._cmd_log_idx = 0 - self._cmd_log = {} # Last `_cmd_log_len' interactions + self._cmd_log = {} # Last '_cmd_log_len' interactions if self.debug >= 1: self._mesg('imaplib version %s' % __version__) self._mesg('new IMAP4 connection, tag=%s' % self.tagpre) @@ -396,7 +396,7 @@ def append(self, mailbox, flags, date_time, message): (typ, [data]) = .append(mailbox, flags, date_time, message) - All args except `message' can be None. + All args except 'message' can be None. """ name = 'APPEND' if not mailbox: @@ -927,7 +927,7 @@ def xatom(self, name, *args): (typ, [data]) = .xatom(name, arg, ...) - Returns response appropriate to extension command `name'. + Returns response appropriate to extension command 'name'. """ name = name.upper() #if not name in self.capabilities: # Let the server decide! @@ -1167,7 +1167,7 @@ def _get_tagged_response(self, tag, expect_bye=False): # Some have reported "unexpected response" exceptions. # Note that ignoring them here causes loops. # Instead, send me details of the unexpected response and - # I'll update the code in `_get_response()'. + # I'll update the code in '_get_response()'. try: self._get_response() @@ -1259,7 +1259,7 @@ def _dump_ur(self, untagged_resp_dict): self._mesg('untagged responses dump:' + '\n\t\t'.join(items)) def _log(self, line): - # Keep log of last `_cmd_log_len' interactions for debugging. + # Keep log of last '_cmd_log_len' interactions for debugging. self._cmd_log[self._cmd_log_idx] = (line, time.time()) self._cmd_log_idx += 1 if self._cmd_log_idx >= self._cmd_log_len: diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 8604000ed77a19..bacee8ba164b48 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -116,7 +116,7 @@ def guess_type(self, url, strict=True): mapped to '.tar.gz'. (This is table-driven too, using the dictionary suffix_map.) - Optional `strict' argument when False adds a bunch of commonly found, + Optional 'strict' argument when False adds a bunch of commonly found, but non-standard types. """ # TODO: Deprecate accepting file paths (in particular path-like objects). @@ -185,9 +185,9 @@ def guess_all_extensions(self, type, strict=True): Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, - but would be mapped to the MIME type `type' by guess_type(). + but would be mapped to the MIME type 'type' by guess_type(). - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ type = type.lower() @@ -204,11 +204,11 @@ def guess_extension(self, type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ extensions = self.guess_all_extensions(type, strict) @@ -314,7 +314,7 @@ def guess_type(url, strict=True): to ".tar.gz". (This is table-driven too, using the dictionary suffix_map). - Optional `strict' argument when false adds a bunch of commonly found, but + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -338,11 +338,11 @@ def guess_all_extensions(type, strict=True): Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -355,10 +355,10 @@ def guess_extension(type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the - MIME type `type' by guess_type(). If no extension can be guessed for - `type', None is returned. + MIME type 'type' by guess_type(). If no extension can be guessed for + 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: diff --git a/Lib/smtplib.py b/Lib/smtplib.py index b3cc68a789a7d8..75163f75781d19 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -105,7 +105,7 @@ class SMTPSenderRefused(SMTPResponseException): """Sender address refused. In addition to the attributes set by on all SMTPResponseException - exceptions, this sets `sender' to the string that the SMTP refused. + exceptions, this sets 'sender' to the string that the SMTP refused. """ def __init__(self, code, msg, sender): @@ -315,7 +315,7 @@ def _get_socket(self, host, port, timeout): def connect(self, host='localhost', port=0, source_address=None): """Connect to a host on a given port. - If the hostname ends with a colon (`:') followed by a number, and + If the hostname ends with a colon (':') followed by a number, and there is no port specified, that suffix will be stripped off and the number interpreted as the port number to use. @@ -346,7 +346,7 @@ def connect(self, host='localhost', port=0, source_address=None): return (code, msg) def send(self, s): - """Send `s' to the server.""" + """Send 's' to the server.""" if self.debuglevel > 0: self._print_debug('send:', repr(s)) if self.sock: diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 5fc6183ffcf93c..f817b57ab1640e 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1217,7 +1217,7 @@ def _create_pax_generic_header(cls, pax_headers, type, encoding): for keyword, value in pax_headers.items(): keyword = keyword.encode("utf-8") if binary: - # Try to restore the original byte representation of `value'. + # Try to restore the original byte representation of 'value'. # Needless to say, that the encoding must match the string. value = value.encode(encoding, "surrogateescape") else: @@ -1663,13 +1663,13 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, errorlevel=None, copybufsize=None, stream=False): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to + """Open an (uncompressed) tar archive 'name'. 'mode' is either 'r' to read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' + file or 'w' to create a new file overwriting an existing one. 'mode' defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. + If 'fileobj' is given, it is used for reading or writing data. If it + can be determined, 'mode' is overridden by 'fileobj's mode. + 'fileobj' is not closed, when TarFile is closed. """ modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} if mode not in modes: @@ -1998,7 +1998,7 @@ def close(self): self.fileobj.close() def getmember(self, name): - """Return a TarInfo object for member `name'. If `name' can not be + """Return a TarInfo object for member 'name'. If 'name' can not be found in the archive, KeyError is raised. If a member occurs more than once in the archive, its last occurrence is assumed to be the most up-to-date version. @@ -2026,9 +2026,9 @@ def getnames(self): def gettarinfo(self, name=None, arcname=None, fileobj=None): """Create a TarInfo object from the result of os.stat or equivalent - on an existing file. The file is either named by `name', or - specified as a file object `fileobj' with a file descriptor. If - given, `arcname' specifies an alternative name for the file in the + on an existing file. The file is either named by 'name', or + specified as a file object 'fileobj' with a file descriptor. If + given, 'arcname' specifies an alternative name for the file in the archive, otherwise, the name is taken from the 'name' attribute of 'fileobj', or the 'name' argument. The name should be a text string. @@ -2124,9 +2124,9 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): return tarinfo def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If `verbose' is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. `members' is optional and must be a subset of the + """Print a table of contents to sys.stdout. If 'verbose' is False, only + the names of the members are printed. If it is True, an 'ls -l'-like + output is produced. 'members' is optional and must be a subset of the list returned by getmembers(). """ # Convert tarinfo type to stat type. @@ -2167,11 +2167,11 @@ def list(self, verbose=True, *, members=None): print() def add(self, name, arcname=None, recursive=True, *, filter=None): - """Add the file `name' to the archive. `name' may be any type of file - (directory, fifo, symbolic link, etc.). If given, `arcname' + """Add the file 'name' to the archive. 'name' may be any type of file + (directory, fifo, symbolic link, etc.). If given, 'arcname' specifies an alternative name for the file in the archive. Directories are added recursively by default. This can be avoided by - setting `recursive' to False. `filter' is a function + setting 'recursive' to False. 'filter' is a function that expects a TarInfo object argument and returns the changed TarInfo object, if it returns None the TarInfo object will be excluded from the archive. @@ -2218,8 +2218,8 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object `tarinfo' to the archive. If `tarinfo' represents - a non zero-size regular file, the `fileobj' argument should be a binary file, + """Add the TarInfo object 'tarinfo' to the archive. If 'tarinfo' represents + a non zero-size regular file, the 'fileobj' argument should be a binary file, and tarinfo.size bytes are read from it and added to the archive. You can create TarInfo objects directly, or by using gettarinfo(). """ @@ -2273,12 +2273,12 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, filter=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). If `numeric_owner` is True, only + directories afterwards. 'path' specifies a different directory + to extract to. 'members' is optional and must be a subset of the + list returned by getmembers(). If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. - The `filter` function will be called on each member just + The 'filter' function will be called on each member just before extraction. It can return a changed TarInfo or None to skip the member. String names of common filters are accepted. @@ -2318,13 +2318,13 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, filter=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. File attributes (owner, - mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` + as possible. 'member' may be a filename or a TarInfo object. You can + specify a different directory using 'path'. File attributes (owner, + mtime, mode) are set unless 'set_attrs' is False. If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. - The `filter` function will be called before extraction. + The 'filter' function will be called before extraction. It can return a changed TarInfo or None to skip the member. String names of common filters are accepted. """ @@ -2389,10 +2389,10 @@ def _handle_fatal_error(self, e): self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) def extractfile(self, member): - """Extract a member from the archive as a file object. `member' may be - a filename or a TarInfo object. If `member' is a regular file or + """Extract a member from the archive as a file object. 'member' may be + a filename or a TarInfo object. If 'member' is a regular file or a link, an io.BufferedReader object is returned. For all other - existing members, None is returned. If `member' does not appear + existing members, None is returned. If 'member' does not appear in the archive, KeyError is raised. """ self._check("r") @@ -2590,7 +2590,7 @@ def chown(self, tarinfo, targetpath, numeric_owner): else: os.chown(targetpath, u, g) except (OSError, OverflowError) as e: - # OverflowError can be raised if an ID doesn't fit in `id_t` + # OverflowError can be raised if an ID doesn't fit in 'id_t' raise ExtractError("could not change owner") from e def chmod(self, tarinfo, targetpath): diff --git a/Lib/test/support/smtpd.py b/Lib/test/support/smtpd.py index 6052232ec2b585..c2e17cad422861 100755 --- a/Lib/test/support/smtpd.py +++ b/Lib/test/support/smtpd.py @@ -7,7 +7,7 @@ --nosetuid -n - This program generally tries to setuid `nobody', unless this flag is + This program generally tries to setuid 'nobody', unless this flag is set. The setuid call will fail if this program is not run as root (in which case, use this flag). @@ -17,7 +17,7 @@ --class classname -c classname - Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by + Use 'classname' as the concrete SMTP proxy class. Uses 'PureProxy' by default. --size limit @@ -39,8 +39,8 @@ Version: %(__version__)s -If localhost is not given then `localhost' is used, and if localport is not -given then 8025 is used. If remotehost is not given then `localhost' is used, +If localhost is not given then 'localhost' is used, and if localport is not +given then 8025 is used. If remotehost is not given then 'localhost' is used, and if remoteport is not given, then 25 is used. """ @@ -672,9 +672,9 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): message to. data is a string containing the entire full text of the message, - headers (if supplied) and all. It has been `de-transparencied' + headers (if supplied) and all. It has been 'de-transparencied' according to RFC 821, Section 4.5.2. In other words, a line - containing a `.' followed by other text has had the leading dot + containing a '.' followed by other text has had the leading dot removed. kwargs is a dictionary containing additional information. It is @@ -685,7 +685,7 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): ['BODY=8BITMIME', 'SMTPUTF8']. 'rcpt_options': same, for the rcpt command. - This function should return None for a normal `250 Ok' response; + This function should return None for a normal '250 Ok' response; otherwise, it should return the desired response string in RFC 821 format. diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index a0884bffe6b0de..c961dadff9fef2 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -1481,7 +1481,7 @@ async def coro(): # wait again only for rewait tasks await barrier.wait() else: - # wait for end of draining state` + # wait for end of draining state await barrier_nowaiting.wait() # wait for other waiting tasks await barrier.wait() @@ -1780,7 +1780,7 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) async def test_abort_barrier_when_exception_then_resetting(self): - # test from threading.Barrier: see `lock_tests.test_abort_and_reset`` + # test from threading.Barrier: see `lock_tests.test_abort_and_reset` barrier1 = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) results1 = [] diff --git a/Lib/trace.py b/Lib/trace.py index 64fc8037e355ee..8550475e3a78c8 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -402,7 +402,7 @@ def __init__(self, count=1, trace=1, countfuncs=0, countcallers=0, @param countfuncs true iff it should just output a list of (filename, modulename, funcname,) for functions that were called at least once; This overrides - `count' and `trace' + 'count' and 'trace' @param ignoremods a list of the names of modules to ignore @param ignoredirs a list of the names of directories to ignore all of the (recursive) contents of @@ -534,7 +534,7 @@ def globaltrace_countfuncs(self, frame, why, arg): def globaltrace_lt(self, frame, why, arg): """Handler for call events. - If the code block being entered is to be ignored, returns `None', + If the code block being entered is to be ignored, returns 'None', else returns self.localtrace. """ if why == 'call': diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index a2634b6164062a..3ef83e263f53b7 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1748,7 +1748,7 @@ def patch( the patch is undone. If `new` is omitted, then the target is replaced with an - `AsyncMock if the patched object is an async function or a + `AsyncMock` if the patched object is an async function or a `MagicMock` otherwise. If `patch` is used as a decorator and `new` is omitted, the created mock is passed in as an extra argument to the decorated function. If `patch` is used as a context manager the created diff --git a/Lib/wsgiref/headers.py b/Lib/wsgiref/headers.py index fab851c5a44430..05d2ba4c664e5e 100644 --- a/Lib/wsgiref/headers.py +++ b/Lib/wsgiref/headers.py @@ -5,7 +5,7 @@ written by Barry Warsaw. """ -# Regular expression that matches `special' characters in parameters, the +# Regular expression that matches 'special' characters in parameters, the # existence of which force quoting of the parameter value. import re tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 31ef9bb1ad925e..e2aaf8bab4913d 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -1745,8 +1745,8 @@ def _open_to_write(self, zinfo, force_zip64=False): def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. You can specify the + as possible. 'member' may be a filename or a ZipInfo object. You can + specify a different directory using 'path'. You can specify the password to decrypt the file using 'pwd'. """ if path is None: @@ -1758,8 +1758,8 @@ def extract(self, member, path=None, pwd=None): def extractall(self, path=None, members=None, pwd=None): """Extract all members from the archive to the current working - directory. `path' specifies a different directory to extract to. - `members' is optional and must be a subset of the list returned + directory. 'path' specifies a different directory to extract to. + 'members' is optional and must be a subset of the list returned by namelist(). You can specify the password to decrypt all files using 'pwd'. """ diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 695ce22f8049df..80fe9cff98509d 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -585,7 +585,7 @@ non-existing elements are considered to be infinite. The interesting\n\ property of a heap is that a[0] is always its smallest element.\n" "\n\ The strange invariant above is meant to be an efficient memory\n\ -representation for a tournament. The numbers below are `k', not a[k]:\n\ +representation for a tournament. The numbers below are 'k', not a[k]:\n\ \n\ 0\n\ \n\ @@ -598,7 +598,7 @@ representation for a tournament. The numbers below are `k', not a[k]:\n\ 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30\n\ \n\ \n\ -In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In\n\ +In the tree above, each cell 'k' is topping '2*k+1' and '2*k+2'. In\n\ a usual binary tournament we see in sports, each cell is the winner\n\ over the two cells it tops, and we can trace the winner down the tree\n\ to see all opponents s/he had. However, in many computer applications\n\ @@ -653,7 +653,7 @@ vanishes, you switch heaps and start a new run. Clever and quite\n\ effective!\n\ \n\ In a word, heaps are useful memory structures to know. I use them in\n\ -a few applications, and I think it is good to keep a `heap' module\n\ +a few applications, and I think it is good to keep a 'heap' module\n\ around. :-)\n" "\n\ --------------------\n\ diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 86a4113dcc16f1..6df6952dfe384f 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -656,7 +656,7 @@ Create a new interpreter and return a unique generated ID.\n\ The caller is responsible for destroying the interpreter before exiting,\n\ typically by using _interpreters.destroy(). This can be managed \n\ automatically by passing \"reqrefs=True\" and then using _incref() and\n\ -_decref()` appropriately.\n\ +_decref() appropriately.\n\ \n\ \"config\" must be a valid interpreter config or the name of a\n\ predefined config (\"isolated\" or \"legacy\"). The default\n\ diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index b5639d5cf10a22..73edd5c3b25553 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -12,7 +12,7 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, "encode($self, /, input, errors=None)\n" "--\n" "\n" -"Return an encoded string version of `input\'.\n" +"Return an encoded string version of \'input\'.\n" "\n" "\'errors\' may be given to set a different error handling scheme. Default is\n" "\'strict\' meaning that encoding errors raise a UnicodeEncodeError. Other possible\n" @@ -682,4 +682,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=ee767a6d93c7108a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f09052c5a28cc6e6 input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 1c671adb4ff188..373518673dd352 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -574,7 +574,7 @@ _multibytecodec.MultibyteCodec.encode input: object errors: str(accept={str, NoneType}) = None -Return an encoded string version of `input'. +Return an encoded string version of 'input'. 'errors' may be given to set a different error handling scheme. Default is 'strict' meaning that encoding errors raise a UnicodeEncodeError. Other possible @@ -586,7 +586,7 @@ static PyObject * _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *input, const char *errors) -/*[clinic end generated code: output=7b26652045ba56a9 input=606d0e128a577bae]*/ +/*[clinic end generated code: output=7b26652045ba56a9 input=2841745b95ed338f]*/ { MultibyteCodec_State state; PyObject *errorcb, *r, *ucvt; diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 343cb91b975038..e23c910d0ac661 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -61,7 +61,7 @@ PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__, "\n" "Parse XML data.\n" "\n" -"`isfinal\' should be true at end of input."); +"\'isfinal\' should be true at end of input."); #define PYEXPAT_XMLPARSER_PARSE_METHODDEF \ {"Parse", _PyCFunction_CAST(pyexpat_xmlparser_Parse), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_Parse__doc__}, @@ -545,4 +545,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=892e48e41f9b6e4b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=51874bfaf4992ba2 input=a9049054013a1b77]*/ diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index f67d480f19de00..8495fe2dd4dd2b 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -754,13 +754,13 @@ pyexpat.xmlparser.Parse Parse XML data. -`isfinal' should be true at end of input. +'isfinal' should be true at end of input. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_Parse_impl(xmlparseobject *self, PyTypeObject *cls, PyObject *data, int isfinal) -/*[clinic end generated code: output=8faffe07fe1f862a input=d0eb2a69fab3b9f1]*/ +/*[clinic end generated code: output=8faffe07fe1f862a input=053e0f047e55c05a]*/ { const char *s; Py_ssize_t slen; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index cd799a926ae63c..0f6435d84c113e 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -52,25 +52,25 @@ static inline PyObject* bytes_get_empty(void) /* - For PyBytes_FromString(), the parameter `str' points to a null-terminated - string containing exactly `size' bytes. + For PyBytes_FromString(), the parameter 'str' points to a null-terminated + string containing exactly 'size' bytes. - For PyBytes_FromStringAndSize(), the parameter `str' is - either NULL or else points to a string containing at least `size' bytes. - For PyBytes_FromStringAndSize(), the string in the `str' parameter does + For PyBytes_FromStringAndSize(), the parameter 'str' is + either NULL or else points to a string containing at least 'size' bytes. + For PyBytes_FromStringAndSize(), the string in the 'str' parameter does not have to be null-terminated. (Therefore it is safe to construct a - substring by calling `PyBytes_FromStringAndSize(origstring, substrlen)'.) - If `str' is NULL then PyBytes_FromStringAndSize() will allocate `size+1' + substring by calling 'PyBytes_FromStringAndSize(origstring, substrlen)'.) + If 'str' is NULL then PyBytes_FromStringAndSize() will allocate 'size+1' bytes (setting the last byte to the null terminating character) and you can - fill in the data yourself. If `str' is non-NULL then the resulting + fill in the data yourself. If 'str' is non-NULL then the resulting PyBytes object must be treated as immutable and you must not fill in nor alter the data yourself, since the strings may be shared. - The PyObject member `op->ob_size', which denotes the number of "extra + The PyObject member 'op->ob_size', which denotes the number of "extra items" in a variable-size object, will contain the number of bytes allocated for string data, not counting the null terminating character. - It is therefore equal to the `size' parameter (for - PyBytes_FromStringAndSize()) or the length of the string in the `str' + It is therefore equal to the 'size' parameter (for + PyBytes_FromStringAndSize()) or the length of the string in the 'str' parameter (for PyBytes_FromString()). */ static PyObject * diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 53f64fc81e7deb..30277aa0c23883 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -200,7 +200,7 @@ Here are some ways to address this challenge: Adding the checks to the concrete API would help make any interpreter switch to OrderedDict less painful for extension modules. However, this won't work. The equivalent C API call to `dict.__setitem__(obj, k, v)` -is 'PyDict_SetItem(obj, k, v)`. This illustrates how subclasses in C call +is `PyDict_SetItem(obj, k, v)`. This illustrates how subclasses in C call the base class's methods, since there is no equivalent of super() in the C API. Calling into Python for parent class API would work, but some extension modules already rely on this feature of the concrete API. From 2b3fb767bea1f96c9e0523f6cc341b40f0fa1ca1 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Wed, 22 May 2024 13:38:56 -0400 Subject: [PATCH 176/903] gh-117657: Fix missing atomic in dict_resize (#119312) Fix missing atomic in dict_resize --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 985a326a176c94..6e1c3b93fd391b 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2003,7 +2003,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, if (oldvalues->embedded) { assert(oldvalues->embedded == 1); assert(oldvalues->valid == 1); - oldvalues->valid = 0; + FT_ATOMIC_STORE_UINT8(oldvalues->valid, 0); } else { free_values(oldvalues, IS_DICT_SHARED(mp)); From baf347d91643a83483bae110092750d39471e0c2 Mon Sep 17 00:00:00 2001 From: "Josh {*()} Rosenberg" <26495692+MojoVampire@users.noreply.github.com> Date: Wed, 22 May 2024 17:45:34 +0000 Subject: [PATCH 177/903] gh-119247: Add macros to use PySequence_Fast safely in free-threaded build (#119315) Add `Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST` and `Py_END_CRITICAL_SECTION_SEQUENCE_FAST` macros and update `str.join` to use them. Also add a regression test that would crash reliably without this patch. --- Include/internal/pycore_critical_section.h | 22 ++++++ Lib/test/test_free_threading/test_str.py | 75 +++++++++++++++++++ ...-05-21-11-35-11.gh-issue-119247.U6n6mh.rst | 4 + Objects/unicodeobject.c | 8 +- 4 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 Lib/test/test_free_threading/test_str.py create mode 100644 Misc/NEWS.d/next/C API/2024-05-21-11-35-11.gh-issue-119247.U6n6mh.rst diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 573d09a09683ef..7bebcdb67c7169 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -108,6 +108,26 @@ extern "C" { _PyCriticalSection2_End(&_cs2); \ } +// Specialized version of critical section locking to safely use +// PySequence_Fast APIs without the GIL. For performance, the argument *to* +// PySequence_Fast() is provided to the macro, not the *result* of +// PySequence_Fast(), which would require an extra test to determine if the +// lock must be acquired. +# define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) \ + { \ + PyObject *_orig_seq = _PyObject_CAST(original); \ + const bool _should_lock_cs = PyList_CheckExact(_orig_seq); \ + _PyCriticalSection _cs; \ + if (_should_lock_cs) { \ + _PyCriticalSection_Begin(&_cs, &_orig_seq->ob_mutex); \ + } + +# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() \ + if (_should_lock_cs) { \ + _PyCriticalSection_End(&_cs); \ + } \ + } + // Asserts that the mutex is locked. The mutex must be held by the // top-most critical section otherwise there's the possibility // that the mutex would be swalled out in some code paths. @@ -137,6 +157,8 @@ extern "C" { # define Py_END_CRITICAL_SECTION() # define Py_BEGIN_CRITICAL_SECTION2(a, b) # define Py_END_CRITICAL_SECTION2() +# define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) +# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* !Py_GIL_DISABLED */ diff --git a/Lib/test/test_free_threading/test_str.py b/Lib/test/test_free_threading/test_str.py new file mode 100644 index 00000000000000..f5e93b7de0aa9b --- /dev/null +++ b/Lib/test/test_free_threading/test_str.py @@ -0,0 +1,75 @@ +import sys +import unittest + +from itertools import cycle +from threading import Event, Thread +from unittest import TestCase + +from test.support import threading_helper + +@threading_helper.requires_working_threading() +class TestStr(TestCase): + def test_racing_join_extend(self): + '''Test joining a string being extended by another thread''' + l = [] + ITERS = 100 + READERS = 10 + done_event = Event() + def writer_func(): + for i in range(ITERS): + l.extend(map(str, range(i))) + l.clear() + done_event.set() + def reader_func(): + while not done_event.is_set(): + ''.join(l) + writer = Thread(target=writer_func) + readers = [] + for x in range(READERS): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + def test_racing_join_replace(self): + ''' + Test joining a string of characters being replaced with ephemeral + strings by another thread. + ''' + l = [*'abcdefg'] + MAX_ORDINAL = 1_000 + READERS = 10 + done_event = Event() + + def writer_func(): + for i, c in zip(cycle(range(len(l))), + map(chr, range(128, MAX_ORDINAL))): + l[i] = c + done_event.set() + + def reader_func(): + while not done_event.is_set(): + ''.join(l) + ''.join(l) + ''.join(l) + ''.join(l) + + writer = Thread(target=writer_func) + readers = [] + for x in range(READERS): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/C API/2024-05-21-11-35-11.gh-issue-119247.U6n6mh.rst b/Misc/NEWS.d/next/C API/2024-05-21-11-35-11.gh-issue-119247.U6n6mh.rst new file mode 100644 index 00000000000000..3b2cdc8cf2dc5c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-21-11-35-11.gh-issue-119247.U6n6mh.rst @@ -0,0 +1,4 @@ +Added ``Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST`` and +``Py_END_CRITICAL_SECTION_SEQUENCE_FAST`` macros to make it possible to use +PySequence_Fast APIs safely when free-threaded, and update str.join to work +without the GIL using them. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 057b417074ebea..480b6713905c70 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -44,6 +44,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_bytesobject.h" // _PyBytes_Repeat() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_codecs.h" // _PyCodec_Lookup() +#include "pycore_critical_section.h" // Py_*_CRITICAL_SECTION_SEQUENCE_FAST #include "pycore_format.h" // F_LJUST #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.fs_codec @@ -9559,13 +9560,14 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) return NULL; } - /* NOTE: the following code can't call back into Python code, - * so we are sure that fseq won't be mutated. - */ + Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(seq); items = PySequence_Fast_ITEMS(fseq); seqlen = PySequence_Fast_GET_SIZE(fseq); res = _PyUnicode_JoinArray(separator, items, seqlen); + + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); + Py_DECREF(fseq); return res; } From c9073eb1a99606df1efeb8959e9f11a8ebc23ae2 Mon Sep 17 00:00:00 2001 From: Michael Vincent <377567+Vynce@users.noreply.github.com> Date: Wed, 22 May 2024 12:59:47 -0500 Subject: [PATCH 178/903] gh-117505: Run ensurepip in isolated env in Windows installer (GH-118257) ensurepip forks a subprocess to run pip itself, but that subprocess only inherits a -I isolated mode flag (see _run_pip() in Lib/ensurepip/__init__.py), not the "-E -s" flags that the installer has been using. This means that parts of ensurepip don't actually run in an isolated environment and can make incorrect decisions based on packages installed in the user site-packages. --- .../Windows/2024-04-24-22-50-33.gh-issue-117505.gcTb_p.rst | 1 + Tools/msi/pip/pip.wxs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-04-24-22-50-33.gh-issue-117505.gcTb_p.rst diff --git a/Misc/NEWS.d/next/Windows/2024-04-24-22-50-33.gh-issue-117505.gcTb_p.rst b/Misc/NEWS.d/next/Windows/2024-04-24-22-50-33.gh-issue-117505.gcTb_p.rst new file mode 100644 index 00000000000000..0931687ecc521c --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-04-24-22-50-33.gh-issue-117505.gcTb_p.rst @@ -0,0 +1 @@ +Fixes an issue with the Windows installer not running ensurepip in a fully isolated environment. This could cause unexpected interactions with the user site-packages. diff --git a/Tools/msi/pip/pip.wxs b/Tools/msi/pip/pip.wxs index 1d8083cad91a56..627c4710a9fdfa 100644 --- a/Tools/msi/pip/pip.wxs +++ b/Tools/msi/pip/pip.wxs @@ -25,8 +25,8 @@ - - + + (&DefaultFeature=3) AND NOT (!DefaultFeature=3) From 2fbea81d646688cf438be1dc0be82112a9ae4325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?u=C4=B1=C9=90=C9=BE=20=CA=9E=20=CA=87=C9=90=C9=AF=C9=90s?= <_@skj.io> Date: Wed, 22 May 2024 13:15:11 -0700 Subject: [PATCH 179/903] gh-70795: Rework RLock documentation (#103853) Attempted to simultaneously reduce verbosity, while more descriptively describing behavior. Fix links (RLock acquire/release previously linking to Lock acquire/release, seems like bad copy pasta). Add a seealso for with-locks. Switch section to use bullet points. --------- Co-authored-by: Alex Waygood Co-authored-by: C.A.M. Gerlach --- Doc/library/threading.rst | 75 ++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 747d39aae03a3e..7b259e22dc7124 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -598,14 +598,25 @@ and "recursion level" in addition to the locked/unlocked state used by primitive locks. In the locked state, some thread owns the lock; in the unlocked state, no thread owns it. -To lock the lock, a thread calls its :meth:`~RLock.acquire` method; this -returns once the thread owns the lock. To unlock the lock, a thread calls -its :meth:`~Lock.release` method. :meth:`~Lock.acquire`/:meth:`~Lock.release` -call pairs may be nested; only the final :meth:`~Lock.release` (the -:meth:`~Lock.release` of the outermost pair) resets the lock to unlocked and -allows another thread blocked in :meth:`~Lock.acquire` to proceed. +Threads call a lock's :meth:`~RLock.acquire` method to lock it, +and its :meth:`~Lock.release` method to unlock it. -Reentrant locks also support the :ref:`context management protocol `. +.. note:: + + Reentrant locks support the :ref:`context management protocol `, + so it is recommended to use :keyword:`with` instead of manually calling + :meth:`~RLock.acquire` and :meth:`~RLock.release` + to handle acquiring and releasing the lock for a block of code. + +RLock's :meth:`~RLock.acquire`/:meth:`~RLock.release` call pairs may be nested, +unlike Lock's :meth:`~Lock.acquire`/:meth:`~Lock.release`. Only the final +:meth:`~RLock.release` (the :meth:`~Lock.release` of the outermost pair) resets +the lock to an unlocked state and allows another thread blocked in +:meth:`~RLock.acquire` to proceed. + +:meth:`~RLock.acquire`/:meth:`~RLock.release` must be used in pairs: each acquire +must have a release in the thread that has acquired the lock. Failing to +call release as many times the lock has been acquired can lead to deadlock. .. class:: RLock() @@ -624,25 +635,41 @@ Reentrant locks also support the :ref:`context management protocol ` Acquire a lock, blocking or non-blocking. - When invoked without arguments: if this thread already owns the lock, increment - the recursion level by one, and return immediately. Otherwise, if another - thread owns the lock, block until the lock is unlocked. Once the lock is - unlocked (not owned by any thread), then grab ownership, set the recursion level - to one, and return. If more than one thread is blocked waiting until the lock - is unlocked, only one at a time will be able to grab ownership of the lock. - There is no return value in this case. + .. seealso:: - When invoked with the *blocking* argument set to ``True``, do the same thing as when - called without arguments, and return ``True``. + :ref:`Using RLock as a context manager ` + Recommended over manual :meth:`!acquire` and :meth:`release` calls + whenever practical. - When invoked with the *blocking* argument set to ``False``, do not block. If a call - without an argument would block, return ``False`` immediately; otherwise, do the - same thing as when called without arguments, and return ``True``. - When invoked with the floating-point *timeout* argument set to a positive - value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. Return ``True`` if the lock has - been acquired, ``False`` if the timeout has elapsed. + When invoked with the *blocking* argument set to ``True`` (the default): + + * If no thread owns the lock, acquire the lock and return immediately. + + * If another thread owns the lock, block until we are able to acquire + lock, or *timeout*, if set to a positive float value. + + * If the same thread owns the lock, acquire the lock again, and + return immediately. This is the difference between :class:`Lock` and + :class:`!RLock`; :class:`Lock` handles this case the same as the previous, + blocking until the lock can be acquired. + + When invoked with the *blocking* argument set to ``False``: + + * If no thread owns the lock, acquire the lock and return immediately. + + * If another thread owns the lock, return immediately. + + * If the same thread owns the lock, acquire the lock again and return + immediately. + + In all cases, if the thread was able to acquire the lock, return ``True``. + If the thread was unable to acquire the lock (i.e. if not blocking or + the timeout was reached) return ``False``. + + If called multiple times, failing to call :meth:`~RLock.release` as many times + may lead to deadlock. Consider using :class:`!RLock` as a context manager rather than + calling acquire/release directly. .. versionchanged:: 3.2 The *timeout* parameter is new. @@ -658,7 +685,7 @@ Reentrant locks also support the :ref:`context management protocol ` Only call this method when the calling thread owns the lock. A :exc:`RuntimeError` is raised if this method is called when the lock is - unlocked. + not acquired. There is no return value. From e12a6780bbb607fce6ecf0ffe4260e5a944676c1 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 23 May 2024 05:30:41 +0900 Subject: [PATCH 180/903] gh-117142: ctypes: Clean up c-analyzer .tsv files (GH-117544) Co-authored-by: Petr Viktorin --- Modules/_ctypes/ctypes.h | 6 --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 44 --------------------- Tools/c-analyzer/cpython/ignored.tsv | 3 -- 3 files changed, 53 deletions(-) diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 20c68134be2804..7784020e9af45a 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -427,14 +427,8 @@ struct basespec { char *adr; }; -extern char basespec_string[]; - extern ffi_type *_ctypes_get_ffi_type(ctypes_state *st, PyObject *obj); -extern char *_ctypes_conversion_encoding; -extern char *_ctypes_conversion_errors; - - extern void _ctypes_free_closure(void *); extern void *_ctypes_alloc_closure(void); diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 1b8cccf80872c8..8b6fe94e3afc52 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -354,37 +354,6 @@ Modules/_testclinic.c - TestClass - ##----------------------- ## static types -Modules/_ctypes/_ctypes.c - PyCArrayType_Type - -Modules/_ctypes/_ctypes.c - PyCArray_Type - -Modules/_ctypes/_ctypes.c - PyCData_Type - -Modules/_ctypes/_ctypes.c - PyCFuncPtrType_Type - -Modules/_ctypes/_ctypes.c - PyCFuncPtr_Type - -Modules/_ctypes/_ctypes.c - PyCPointerType_Type - -Modules/_ctypes/_ctypes.c - PyCPointer_Type - -Modules/_ctypes/_ctypes.c - PyCSimpleType_Type - -Modules/_ctypes/_ctypes.c - PyCStructType_Type - -Modules/_ctypes/_ctypes.c - Simple_Type - -Modules/_ctypes/_ctypes.c - Struct_Type - -Modules/_ctypes/_ctypes.c - UnionType_Type - -Modules/_ctypes/_ctypes.c - Union_Type - -Modules/_ctypes/callproc.c - PyCArg_Type - -Modules/_ctypes/ctypes.h - PyCArg_Type - -Modules/_ctypes/ctypes.h - PyCArrayType_Type - -Modules/_ctypes/ctypes.h - PyCArray_Type - -Modules/_ctypes/ctypes.h - PyCData_Type - -Modules/_ctypes/ctypes.h - PyCFuncPtrType_Type - -Modules/_ctypes/ctypes.h - PyCFuncPtr_Type - -Modules/_ctypes/ctypes.h - PyCPointerType_Type - -Modules/_ctypes/ctypes.h - PyCPointer_Type - -Modules/_ctypes/ctypes.h - PyCSimpleType_Type - -Modules/_ctypes/ctypes.h - PyCStgDict_Type - -Modules/_ctypes/ctypes.h - PyCStructType_Type - -Modules/_ctypes/ctypes.h - PyExc_ArgError - -Modules/_ctypes/ctypes.h - _ctypes_conversion_encoding - -Modules/_ctypes/ctypes.h - _ctypes_conversion_errors - -Modules/_ctypes/ctypes.h - _ctypes_ptrtype_cache - -Modules/_ctypes/ctypes.h - basespec_string - -Modules/_ctypes/stgdict.c - PyCStgDict_Type - Modules/_cursesmodule.c - PyCursesWindow_Type - Modules/_datetimemodule.c - PyDateTime_DateTimeType - Modules/_datetimemodule.c - PyDateTime_DateType - @@ -410,32 +379,19 @@ Modules/_tkinter.c - Tktt_Type - Modules/xxlimited_35.c - Xxo_Type - ## exception types -Modules/_ctypes/_ctypes.c - PyExc_ArgError - Modules/_cursesmodule.c - PyCursesError - Modules/_tkinter.c - Tkinter_TclError - Modules/xxlimited_35.c - ErrorObject - Modules/xxmodule.c - ErrorObject - -##----------------------- -## cached - initialized once - -## manually cached PyUnicodeOjbect -Modules/_ctypes/callproc.c _ctypes_get_errobj error_object_name - -Modules/_ctypes/_ctypes.c CreateSwappedType swapped_suffix - - ##----------------------- ## other ## initialized once -Modules/_ctypes/_ctypes.c - _unpickle - -Modules/_ctypes/_ctypes.c PyCArrayType_from_ctype array_cache - Modules/_cursesmodule.c - ModDict - Modules/_datetimemodule.c datetime_strptime module - ## state -Modules/_ctypes/_ctypes.c - _ctypes_ptrtype_cache - -Modules/_ctypes/_ctypes.c - global_state - -Modules/_ctypes/ctypes.h - global_state - Modules/_datetimemodule.c - _datetime_global_state - Modules/_tkinter.c - tcl_lock - Modules/_tkinter.c - excInCmd - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index a1c1553d8ddc10..a3bdf0396fd3e1 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -395,8 +395,6 @@ Python/optimizer.c - cold_exits_initialized - ##----------------------- ## test code -Modules/_ctypes/_ctypes_test.c - _ctypes_test_slots - -Modules/_ctypes/_ctypes_test.c - _ctypes_testmodule - Modules/_ctypes/_ctypes_test.c - _xxx_lib - Modules/_ctypes/_ctypes_test.c - an_integer - Modules/_ctypes/_ctypes_test.c - bottom - @@ -404,7 +402,6 @@ Modules/_ctypes/_ctypes_test.c - last_tf_arg_s - Modules/_ctypes/_ctypes_test.c - last_tf_arg_u - Modules/_ctypes/_ctypes_test.c - last_tfrsuv_arg - Modules/_ctypes/_ctypes_test.c - left - -Modules/_ctypes/_ctypes_test.c - module_methods - Modules/_ctypes/_ctypes_test.c - my_eggs - Modules/_ctypes/_ctypes_test.c - my_spams - Modules/_ctypes/_ctypes_test.c - right - From 14b063cbf1bb11a489d04a31f277edba0fc8893c Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 22 May 2024 17:02:33 -0400 Subject: [PATCH 181/903] gh-111201: Use calc_complete_screen after bracketed paste in PyREPL (#119432) --- Lib/_pyrepl/commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 33c6564f640421..ed977f84baac4e 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -475,3 +475,4 @@ def do(self) -> None: self.reader.paste_mode = False self.reader.in_bracketed_paste = False self.reader.dirty = True + self.reader.calc_screen = self.reader.calc_complete_screen From 9b422fc6af87b81812aaf3010c004eb27c4dc280 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 22 May 2024 23:05:26 +0200 Subject: [PATCH 182/903] gh-119396: Optimize PyUnicode_FromFormat() UTF-8 decoder (#119398) Add unicode_decode_utf8_writer() to write directly characters into a _PyUnicodeWriter writer: avoid the creation of a temporary string. Optimize PyUnicode_FromFormat() by using the new unicode_decode_utf8_writer(). Rename unicode_fromformat_write_cstr() to unicode_fromformat_write_utf8(). Microbenchmark on the code: return PyUnicode_FromFormat( "%s %s %s %s %s.", "format", "multiple", "utf8", "short", "strings"); Result: 620 ns +- 8 ns -> 382 ns +- 2 ns: 1.62x faster. --- Objects/unicodeobject.c | 203 ++++++++++++++++++++++++++++------------ 1 file changed, 141 insertions(+), 62 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 480b6713905c70..afff37467caf32 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -202,6 +202,11 @@ static PyObject * unicode_decode_utf8(const char *s, Py_ssize_t size, _Py_error_handler error_handler, const char *errors, Py_ssize_t *consumed); +static int +unicode_decode_utf8_writer(_PyUnicodeWriter *writer, + const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed); #ifdef Py_DEBUG static inline int unicode_is_finalizing(void); static int unicode_is_singleton(PyObject *unicode); @@ -2377,14 +2382,11 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, } static int -unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, +unicode_fromformat_write_utf8(_PyUnicodeWriter *writer, const char *str, Py_ssize_t width, Py_ssize_t precision, int flags) { /* UTF-8 */ Py_ssize_t length; - PyObject *unicode; - int res; - if (precision == -1) { length = strlen(str); } @@ -2394,11 +2396,19 @@ unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, length++; } } - unicode = PyUnicode_DecodeUTF8Stateful(str, length, "replace", NULL); + + if (width < 0) { + return unicode_decode_utf8_writer(writer, str, length, + _Py_ERROR_REPLACE, "replace", NULL); + } + + PyObject *unicode = PyUnicode_DecodeUTF8Stateful(str, length, + "replace", NULL); if (unicode == NULL) return -1; - res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); + int res = unicode_fromformat_write_str(writer, unicode, + width, -1, flags); Py_DECREF(unicode); return res; } @@ -2700,7 +2710,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, else { /* UTF-8 */ const char *s = va_arg(*vargs, const char*); - if (unicode_fromformat_write_cstr(writer, s, width, precision, flags) < 0) + if (unicode_fromformat_write_utf8(writer, s, width, precision, flags) < 0) return NULL; } break; @@ -2739,7 +2749,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, } else { assert(str != NULL); - if (unicode_fromformat_write_cstr(writer, str, width, precision, flags) < 0) + if (unicode_fromformat_write_utf8(writer, str, width, precision, flags) < 0) return NULL; } break; @@ -4737,46 +4747,14 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) return p - start; } -static PyObject * -unicode_decode_utf8(const char *s, Py_ssize_t size, - _Py_error_handler error_handler, const char *errors, - Py_ssize_t *consumed) -{ - if (size == 0) { - if (consumed) - *consumed = 0; - _Py_RETURN_UNICODE_EMPTY(); - } - - /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 && (unsigned char)s[0] < 128) { - if (consumed) { - *consumed = 1; - } - return get_latin1_char((unsigned char)s[0]); - } - - const char *starts = s; - const char *end = s + size; - - // fast path: try ASCII string. - PyObject *u = PyUnicode_New(size, 127); - if (u == NULL) { - return NULL; - } - s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); - if (s == end) { - if (consumed) { - *consumed = size; - } - return u; - } - - // Use _PyUnicodeWriter after fast path is failed. - _PyUnicodeWriter writer; - _PyUnicodeWriter_InitWithBuffer(&writer, u); - writer.pos = s - starts; +static int +unicode_decode_utf8_impl(_PyUnicodeWriter *writer, + const char *starts, const char *s, const char *end, + _Py_error_handler error_handler, + const char *errors, + Py_ssize_t *consumed) +{ Py_ssize_t startinpos, endinpos; const char *errmsg = ""; PyObject *error_handler_obj = NULL; @@ -4784,18 +4762,18 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, while (s < end) { Py_UCS4 ch; - int kind = writer.kind; + int kind = writer->kind; if (kind == PyUnicode_1BYTE_KIND) { - if (PyUnicode_IS_ASCII(writer.buffer)) - ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos); + if (PyUnicode_IS_ASCII(writer->buffer)) + ch = asciilib_utf8_decode(&s, end, writer->data, &writer->pos); else - ch = ucs1lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs1lib_utf8_decode(&s, end, writer->data, &writer->pos); } else if (kind == PyUnicode_2BYTE_KIND) { - ch = ucs2lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs2lib_utf8_decode(&s, end, writer->data, &writer->pos); } else { assert(kind == PyUnicode_4BYTE_KIND); - ch = ucs4lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs4lib_utf8_decode(&s, end, writer->data, &writer->pos); } switch (ch) { @@ -4826,7 +4804,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, endinpos = startinpos + ch - 1; break; default: - if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) + // ch doesn't fit into kind, so change the buffer kind to write + // the character + if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) goto onError; continue; } @@ -4840,7 +4820,7 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, break; case _Py_ERROR_REPLACE: - if (_PyUnicodeWriter_WriteCharInline(&writer, 0xfffd) < 0) + if (_PyUnicodeWriter_WriteCharInline(writer, 0xfffd) < 0) goto onError; s += (endinpos - startinpos); break; @@ -4849,13 +4829,13 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, { Py_ssize_t i; - if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + if (_PyUnicodeWriter_PrepareKind(writer, PyUnicode_2BYTE_KIND) < 0) goto onError; for (i=startinpos; ikind, writer->data, writer->pos, ch + 0xdc00); - writer.pos++; + writer->pos++; } s += (endinpos - startinpos); break; @@ -4866,8 +4846,13 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, errors, &error_handler_obj, "utf-8", errmsg, &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) + writer)) { goto onError; + } + + if (_PyUnicodeWriter_Prepare(writer, end - s, 127) < 0) { + return -1; + } } } @@ -4877,13 +4862,107 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return 0; onError: Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - _PyUnicodeWriter_Dealloc(&writer); - return NULL; + return -1; +} + + +static PyObject * +unicode_decode_utf8(const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed) +{ + if (size == 0) { + if (consumed) { + *consumed = 0; + } + _Py_RETURN_UNICODE_EMPTY(); + } + + /* ASCII is equivalent to the first 128 ordinals in Unicode. */ + if (size == 1 && (unsigned char)s[0] < 128) { + if (consumed) { + *consumed = 1; + } + return get_latin1_char((unsigned char)s[0]); + } + + // fast path: try ASCII string. + const char *starts = s; + const char *end = s + size; + PyObject *u = PyUnicode_New(size, 127); + if (u == NULL) { + return NULL; + } + Py_ssize_t decoded = ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); + if (decoded == size) { + if (consumed) { + *consumed = size; + } + return u; + } + s += decoded; + size -= decoded; + + // Use _PyUnicodeWriter after fast path is failed. + _PyUnicodeWriter writer; + _PyUnicodeWriter_InitWithBuffer(&writer, u); + writer.pos = decoded; + + if (unicode_decode_utf8_impl(&writer, starts, s, end, + error_handler, errors, + consumed) < 0) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); +} + + +static int +unicode_decode_utf8_writer(_PyUnicodeWriter *writer, + const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed) +{ + if (size == 0) { + if (consumed) { + *consumed = 0; + } + return 0; + } + + // fast path: try ASCII string. + if (_PyUnicodeWriter_Prepare(writer, size, 127) < 0) { + return -1; + } + + const char *starts = s; + const char *end = s + size; + Py_ssize_t decoded = 0; + Py_UCS1 *dest = (Py_UCS1*)writer->data + writer->pos * writer->kind; + if (writer->kind == PyUnicode_1BYTE_KIND + && _Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)) + { + decoded = ascii_decode(s, end, dest); + writer->pos += decoded; + + if (decoded == size) { + if (consumed) { + *consumed = size; + } + return 0; + } + s += decoded; + size -= decoded; + } + + return unicode_decode_utf8_impl(writer, starts, s, end, + error_handler, errors, consumed); } From e3bf5381fd056d0bbdd775463e3724aab2012e45 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Wed, 22 May 2024 15:03:32 -0700 Subject: [PATCH 183/903] gh-119434: Fix culmitive errors in wrapping as lines proceed (#119435) Fix culmitive errors in wrapping as lines proceed --- Lib/_pyrepl/reader.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 768d45a045e3be..81df0c925ee6cb 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -307,7 +307,8 @@ def calc_complete_screen(self) -> list[str]: screen.append(prompt + l) screeninfo.append((lp, l2)) else: - for i in range(wrapcount + 1): + i = 0 + while l: prelen = lp if i == 0 else 0 index_to_wrap_before = 0 column = 0 @@ -317,12 +318,17 @@ def calc_complete_screen(self) -> list[str]: index_to_wrap_before += 1 column += character_width pre = prompt if i == 0 else "" - post = "\\" if i != wrapcount else "" - after = [1] if i != wrapcount else [] + if len(l) > index_to_wrap_before: + post = "\\" + after = [1] + else: + post = "" + after = [] screen.append(pre + l[:index_to_wrap_before] + post) screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) l = l[index_to_wrap_before:] l2 = l2[index_to_wrap_before:] + i += 1 self.screeninfo = screeninfo self.cxy = self.pos2xy() if self.msg and self.msg_at_bottom: From 07df93de73b6bdd4f80cd9e29834d761a0882622 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Wed, 22 May 2024 18:25:08 -0500 Subject: [PATCH 184/903] gh-119105: difflib.py Differ.compare is too slow [for degenerate cases] (#119376) Track all pairs achieving the best ratio in Differ(). This repairs the "very deep recursion and cubic time" bad cases in a way that preserves previous output. --- Lib/difflib.py | 128 +++++++++++++++++++++------------------ Lib/test/test_difflib.py | 20 ++++++ 2 files changed, 89 insertions(+), 59 deletions(-) diff --git a/Lib/difflib.py b/Lib/difflib.py index 54ca33d5615f8d..79b446c2afbdc6 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -908,29 +908,34 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): + abcdefGhijkl ? ^ ^ ^ """ - - # don't synch up unless the lines have a similarity score of at - # least cutoff; best_ratio tracks the best score seen so far - # best_ratio is a tuple storing the best .ratio() seen so far, and - # a measure of how far the indices are from their index range - # midpoints. The latter is used to resolve ratio ties. Favoring - # indices near the midpoints tends to cut the ranges in half. Else, - # if there are many pairs with the best ratio, recursion can grow - # very deep, and runtime becomes cubic. See: + from operator import ge, gt + # Don't synch up unless the lines have a similarity score of at + # least cutoff; best_ratio tracks the best score seen so far. + # Keep track of all index pairs achieving the best ratio and + # deal with them here. Previously only the smallest pair was + # handled here, and if there are many pairs with the best ratio, + # recursion could grow very deep, and runtime cubic. See: # https://github.com/python/cpython/issues/119105 - best_ratio, cutoff = (0.74, 0), 0.75 + best_ratio, cutoff = 0.74, 0.75 cruncher = SequenceMatcher(self.charjunk) eqi, eqj = None, None # 1st indices of equal lines (if any) + # List of index pairs achieving best_ratio. Strictly increasing + # in both index positions. + max_pairs = [] + maxi = -1 # `i` index of last pair in max_pairs # search for the pair that matches best without being identical - # (identical lines must be junk lines, & we don't want to synch up - # on junk -- unless we have to) - amid = (alo + ahi - 1) / 2 - bmid = (blo + bhi - 1) / 2 + # (identical lines must be junk lines, & we don't want to synch + # up on junk -- unless we have to) + crqr = cruncher.real_quick_ratio + cqr = cruncher.quick_ratio + cr = cruncher.ratio for j in range(blo, bhi): bj = b[j] cruncher.set_seq2(bj) - weight_j = - abs(j - bmid) + # Find new best, if possible. Else search for the smallest i + # (if any) > maxi that equals the best ratio + search_equal = True for i in range(alo, ahi): ai = a[i] if ai == bj: @@ -938,65 +943,70 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): eqi, eqj = i, j continue cruncher.set_seq1(ai) - # weight is used to balance the recursion by prioritizing - # i and j in the middle of their ranges - weight = weight_j - abs(i - amid) # computing similarity is expensive, so use the quick # upper bounds first -- have seen this speed up messy # compares by a factor of 3. - # note that ratio() is only expensive to compute the first - # time it's called on a sequence pair; the expensive part - # of the computation is cached by cruncher - if (cruncher.real_quick_ratio(), weight) > best_ratio and \ - (cruncher.quick_ratio(), weight) > best_ratio and \ - (cruncher.ratio(), weight) > best_ratio: - best_ratio, best_i, best_j = (cruncher.ratio(), weight), i, j - best_ratio, _ = best_ratio + cmp = ge if search_equal and i > maxi else gt + if (cmp(crqr(), best_ratio) + and cmp(cqr(), best_ratio) + and cmp((ratio := cr()), best_ratio)): + if ratio > best_ratio: + best_ratio = ratio + max_pairs.clear() + else: + assert best_ratio == ratio and search_equal + assert i > maxi + max_pairs.append((i, j)) + maxi = i + search_equal = False if best_ratio < cutoff: + assert not max_pairs # no non-identical "pretty close" pair if eqi is None: # no identical pair either -- treat it as a straight replace yield from self._plain_replace(a, alo, ahi, b, blo, bhi) return # no close pair, but an identical pair -- synch up on that - best_i, best_j, best_ratio = eqi, eqj, 1.0 + max_pairs = [(eqi, eqj)] else: # there's a close pair, so forget the identical pair (if any) + assert max_pairs eqi = None - # a[best_i] very similar to b[best_j]; eqi is None iff they're not - # identical - - # pump out diffs from before the synch point - yield from self._fancy_helper(a, alo, best_i, b, blo, best_j) - - # do intraline marking on the synch pair - aelt, belt = a[best_i], b[best_j] - if eqi is None: - # pump out a '-', '?', '+', '?' quad for the synched lines - atags = btags = "" - cruncher.set_seqs(aelt, belt) - for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): - la, lb = ai2 - ai1, bj2 - bj1 - if tag == 'replace': - atags += '^' * la - btags += '^' * lb - elif tag == 'delete': - atags += '-' * la - elif tag == 'insert': - btags += '+' * lb - elif tag == 'equal': - atags += ' ' * la - btags += ' ' * lb - else: - raise ValueError('unknown tag %r' % (tag,)) - yield from self._qformat(aelt, belt, atags, btags) - else: - # the synch pair is identical - yield ' ' + aelt + last_i, last_j = alo, blo + for this_i, this_j in max_pairs: + # pump out diffs from before the synch point + yield from self._fancy_helper(a, last_i, this_i, + b, last_j, this_j) + # do intraline marking on the synch pair + aelt, belt = a[this_i], b[this_j] + if eqi is None: + # pump out a '-', '?', '+', '?' quad for the synched lines + atags = btags = "" + cruncher.set_seqs(aelt, belt) + for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): + la, lb = ai2 - ai1, bj2 - bj1 + if tag == 'replace': + atags += '^' * la + btags += '^' * lb + elif tag == 'delete': + atags += '-' * la + elif tag == 'insert': + btags += '+' * lb + elif tag == 'equal': + atags += ' ' * la + btags += ' ' * lb + else: + raise ValueError('unknown tag %r' % (tag,)) + yield from self._qformat(aelt, belt, atags, btags) + else: + # the synch pair is identical + yield ' ' + aelt + last_i, last_j = this_i + 1, this_j + 1 - # pump out diffs from after the synch point - yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi) + # pump out diffs from after the last synch point + yield from self._fancy_helper(a, last_i, ahi, + b, last_j, bhi) def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 6afd90af8442ad..bf6e5b1152b4a2 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -272,6 +272,26 @@ def test_make_file_usascii_charset_with_nonascii_input(self): self.assertIn('content="text/html; charset=us-ascii"', output) self.assertIn('ımplıcıt', output) +class TestDiffer(unittest.TestCase): + def test_close_matches_aligned(self): + # Of the 4 closely matching pairs, we want 1 to match with 3, + # and 2 with 4, to align with a "top to bottom" mental model. + a = ["cat\n", "dog\n", "close match 1\n", "close match 2\n"] + b = ["close match 3\n", "close match 4\n", "kitten\n", "puppy\n"] + m = difflib.Differ().compare(a, b) + self.assertEqual(list(m), + ['- cat\n', + '- dog\n', + '- close match 1\n', + '? ^\n', + '+ close match 3\n', + '? ^\n', + '- close match 2\n', + '? ^\n', + '+ close match 4\n', + '? ^\n', + '+ kitten\n', + '+ puppy\n']) class TestOutputFormat(unittest.TestCase): def test_tab_delimiter(self): From 43807fec8d38a0400c7967baad587da3e89f8dcc Mon Sep 17 00:00:00 2001 From: Ayato Hayashi Date: Thu, 23 May 2024 11:11:11 +0900 Subject: [PATCH 185/903] Tiny fix: Update link for HAC algorithm (gh-118546) --- Objects/longobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index ce3fd6b711dcd4..b0456a311409bf 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3795,7 +3795,7 @@ x_mul(PyLongObject *a, PyLongObject *b) memset(z->long_value.ob_digit, 0, _PyLong_DigitCount(z) * sizeof(digit)); if (a == b) { /* Efficient squaring per HAC, Algorithm 14.16: - * http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf + * https://cacr.uwaterloo.ca/hac/about/chap14.pdf * Gives slightly less than a 2x speedup when a == b, * via exploiting that each entry in the multiplication * pyramid appears twice (except for the size_a squares). @@ -5003,7 +5003,7 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) } else if (i <= HUGE_EXP_CUTOFF / PyLong_SHIFT ) { /* Left-to-right binary exponentiation (HAC Algorithm 14.79) */ - /* http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf */ + /* https://cacr.uwaterloo.ca/hac/about/chap14.pdf */ /* Find the first significant exponent bit. Search right to left * because we're primarily trying to cut overhead for small powers. From e3f5a4455346747fe98f6aeefc74b46048d09bda Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Thu, 23 May 2024 14:41:33 +0800 Subject: [PATCH 186/903] Fix typos in what's new documentation (#119448) --- Misc/NEWS.d/3.13.0a1.rst | 2 +- Misc/NEWS.d/3.13.0a2.rst | 2 +- Misc/NEWS.d/3.13.0a3.rst | 8 ++++---- Misc/NEWS.d/3.13.0a4.rst | 2 +- Misc/NEWS.d/3.13.0a5.rst | 2 +- Misc/NEWS.d/3.13.0a6.rst | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 0ff50e07606aa4..9a321f779c24ff 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -1964,7 +1964,7 @@ debugging. .. nonce: LCxiau .. section: Library -Fix :func:`termios.tcsetattr` bug that was overwritting existing errors +Fix :func:`termios.tcsetattr` bug that was overwriting existing errors during parsing integers from ``term`` list. .. diff --git a/Misc/NEWS.d/3.13.0a2.rst b/Misc/NEWS.d/3.13.0a2.rst index 2480ee8202d0f4..c6b2b1b263ffab 100644 --- a/Misc/NEWS.d/3.13.0a2.rst +++ b/Misc/NEWS.d/3.13.0a2.rst @@ -974,7 +974,7 @@ pattern. .. nonce: 6ah-aw .. section: Library -Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which retuns the +Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. .. diff --git a/Misc/NEWS.d/3.13.0a3.rst b/Misc/NEWS.d/3.13.0a3.rst index c2fe9dc1ad5aa0..2c660192dcd5b3 100644 --- a/Misc/NEWS.d/3.13.0a3.rst +++ b/Misc/NEWS.d/3.13.0a3.rst @@ -609,7 +609,7 @@ with the documentation) :func:`asyncio.Condition.wait()` now re-raises the same :exc:`CancelledError` instance that may have caused it to be interrupted. -Fixed race condition in :func:`asyncio.Semaphore.aquire` when interrupted +Fixed race condition in :func:`asyncio.Semaphore.acquire` when interrupted with a :exc:`CancelledError`. .. @@ -928,7 +928,7 @@ on Windows. .. section: Library Fix :func:`shutil.copymode` and :func:`shutil.copystat` on Windows. -Previously they worked differenly if *dst* is a symbolic link: they modified +Previously they worked differently if *dst* is a symbolic link: they modified the permission bits of *dst* itself rather than the file it points to if *follow_symlinks* is true or *src* is not a symbolic link, and did not modify the permission bits if *follow_symlinks* is false and *src* is a @@ -1550,7 +1550,7 @@ addresses are encountered instead of potentially inaccurate values. Add optional *strict* parameter to these two functions: use ``strict=False`` to get the old behavior, accept malformed inputs. ``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check if the *strict* -paramater is available. Patch by Thomas Dwyer and Victor Stinner to improve +parameter is available. Patch by Thomas Dwyer and Victor Stinner to improve the :cve:`2023-27043` fix. .. @@ -1615,7 +1615,7 @@ method of :class:`itertools.pairwise`. .. section: Library Small (10 - 20%) and trivial performance improvement of -:func:`urrlib.request.getproxies_environment`, typically useful when there +:func:`urllib.request.getproxies_environment`, typically useful when there are many environment variables to go over. .. diff --git a/Misc/NEWS.d/3.13.0a4.rst b/Misc/NEWS.d/3.13.0a4.rst index 39af0534cf8fb5..5efc244c6086cc 100644 --- a/Misc/NEWS.d/3.13.0a4.rst +++ b/Misc/NEWS.d/3.13.0a4.rst @@ -1181,7 +1181,7 @@ configure.ac. .. nonce: XcEXEZ .. section: Build -configure and Makefile were refactored to accomodate framework builds on +configure and Makefile were refactored to accommodate framework builds on Apple platforms other than macOS. .. diff --git a/Misc/NEWS.d/3.13.0a5.rst b/Misc/NEWS.d/3.13.0a5.rst index 55dee59827ad8f..6d74c6bc5c4d55 100644 --- a/Misc/NEWS.d/3.13.0a5.rst +++ b/Misc/NEWS.d/3.13.0a5.rst @@ -962,7 +962,7 @@ On Windows, commas passed in arguments to ``Tools\buildbot\test.bat`` and .. nonce: -dlzfI .. section: Tests -Fix translation of exception hander targets by +Fix translation of exception handler targets by ``_testinternalcapi.optimize_cfg``. .. diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst index 06807b396ed5da..4d44bc664ef8b0 100644 --- a/Misc/NEWS.d/3.13.0a6.rst +++ b/Misc/NEWS.d/3.13.0a6.rst @@ -913,7 +913,7 @@ call. .. nonce: IMLi6K .. section: Documentation -Remove compatibilty references to Emscripten. +Remove compatibility references to Emscripten. .. From c85e3526736d1cf8226686fdf4f5117e105a7b13 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 23 May 2024 05:21:53 -0400 Subject: [PATCH 187/903] gh-119431: fix refleak in test_monitoring (#119444) --- Python/instrumentation.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3d78214738e66b..9095fb981b7981 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -2724,5 +2724,6 @@ _PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelik } PyObject *args[4] = { NULL, NULL, NULL, exc }; int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + Py_DECREF(exc); return exception_event_teardown(err, NULL); } From 406ffb5293a8c9ca315bf63de1ee36a9b33f9aaf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 May 2024 11:06:10 +0100 Subject: [PATCH 188/903] GH-117195: Avoid assertion error in `object.__sizeof__` (GH-117220) --- Lib/test/test_long.py | 2 ++ .../2024-03-25-15-07-01.gh-issue-117195.OWakgD.rst | 2 ++ Objects/typeobject.c | 7 +++++-- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-25-15-07-01.gh-issue-117195.OWakgD.rst diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index d299c34cec076d..41b973da2c7df0 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1639,6 +1639,8 @@ class MyInt(int): MyInt.__basicsize__ + MyInt.__itemsize__ * ndigits ) + # GH-117195 -- This shouldn't crash + object.__sizeof__(1) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-25-15-07-01.gh-issue-117195.OWakgD.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-25-15-07-01.gh-issue-117195.OWakgD.rst new file mode 100644 index 00000000000000..ae1e5acc5c333b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-25-15-07-01.gh-issue-117195.OWakgD.rst @@ -0,0 +1,2 @@ +Avoid assertion failure for debug builds when calling +``object.__sizeof__(1)`` diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9f000d8c193bc5..11f9c570ac4971 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7144,8 +7144,11 @@ object___sizeof___impl(PyObject *self) res = 0; isize = Py_TYPE(self)->tp_itemsize; - if (isize > 0) - res = Py_SIZE(self) * isize; + if (isize > 0) { + /* This assumes that ob_size is valid if tp_itemsize is not 0, + which isn't true for PyLongObject. */ + res = _PyVarObject_CAST(self)->ob_size * isize; + } res += Py_TYPE(self)->tp_basicsize; return PyLong_FromSsize_t(res); From a192547dfe7c4f184cc8b579c3eff2f61f642483 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 23 May 2024 18:01:37 +0200 Subject: [PATCH 189/903] gh-117142: Slightly hacky fix for memory leak of StgInfo (GH-119424) Add a funciton that inlines PyObject_GetTypeData and skips type-checking, so it doesn't need access to the CType_Type object. This will break if the memory layout changes, but should be an acceptable solution to enable ctypes in subinterpreters in Python 3.13. Mark _ctypes as safe for multiple interpreters Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> --- Modules/_ctypes/_ctypes.c | 70 +++++++++++++++++---------------------- Modules/_ctypes/ctypes.h | 28 ++++++++-------- 2 files changed, 45 insertions(+), 53 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 574cb8014493c4..6c1e5f58b95657 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -454,20 +454,17 @@ class _ctypes.CType_Type "PyObject *" "clinic_state()->CType_Type" static int CType_Type_traverse(PyObject *self, visitproc visit, void *arg) { - ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); - if (st && st->PyCType_Type) { - StgInfo *info; - if (PyStgInfo_FromType(st, self, &info) < 0) { - PyErr_WriteUnraisable(self); - } - if (info) { - Py_VISIT(info->proto); - Py_VISIT(info->argtypes); - Py_VISIT(info->converters); - Py_VISIT(info->restype); - Py_VISIT(info->checker); - Py_VISIT(info->module); - } + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + Py_VISIT(info->proto); + Py_VISIT(info->argtypes); + Py_VISIT(info->converters); + Py_VISIT(info->restype); + Py_VISIT(info->checker); + Py_VISIT(info->module); } Py_VISIT(Py_TYPE(self)); return PyType_Type.tp_traverse(self, visit, arg); @@ -488,15 +485,12 @@ ctype_clear_stginfo(StgInfo *info) static int CType_Type_clear(PyObject *self) { - ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); - if (st && st->PyCType_Type) { - StgInfo *info; - if (PyStgInfo_FromType(st, self, &info) < 0) { - PyErr_WriteUnraisable(self); - } - if (info) { - ctype_clear_stginfo(info); - } + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + ctype_clear_stginfo(info); } return PyType_Type.tp_clear(self); } @@ -504,22 +498,20 @@ CType_Type_clear(PyObject *self) static void CType_Type_dealloc(PyObject *self) { - ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); - if (st && st->PyCType_Type) { - StgInfo *info; - if (PyStgInfo_FromType(st, self, &info) < 0) { - PyErr_WriteUnraisable(self); - } - if (info) { - PyMem_Free(info->ffi_type_pointer.elements); - info->ffi_type_pointer.elements = NULL; - PyMem_Free(info->format); - info->format = NULL; - PyMem_Free(info->shape); - info->shape = NULL; - ctype_clear_stginfo(info); - } + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + PyMem_Free(info->ffi_type_pointer.elements); + info->ffi_type_pointer.elements = NULL; + PyMem_Free(info->format); + info->format = NULL; + PyMem_Free(info->shape); + info->shape = NULL; + ctype_clear_stginfo(info); } + PyTypeObject *tp = Py_TYPE(self); PyType_Type.tp_dealloc(self); Py_DECREF(tp); @@ -5947,7 +5939,7 @@ module_free(void *module) static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _ctypes_mod_exec}, - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7784020e9af45a..423120f3460113 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -101,20 +101,6 @@ get_module_state_by_def(PyTypeObject *cls) return get_module_state(mod); } -static inline ctypes_state * -get_module_state_by_def_final(PyTypeObject *cls) -{ - if (cls->tp_mro == NULL) { - return NULL; - } - PyObject *mod = PyType_GetModuleByDef(cls, &_ctypesmodule); - if (mod == NULL) { - PyErr_Clear(); - return NULL; - } - return get_module_state(mod); -} - extern PyType_Spec carg_spec; extern PyType_Spec cfield_spec; @@ -502,6 +488,20 @@ PyStgInfo_FromAny(ctypes_state *state, PyObject *obj, StgInfo **result) return _stginfo_from_type(state, Py_TYPE(obj), result); } +/* A variant of PyStgInfo_FromType that doesn't need the state, + * so it can be called from finalization functions when the module + * state is torn down. Does no checks; cannot fail. + * This inlines the current implementation PyObject_GetTypeData, + * so it might break in the future. + */ +static inline StgInfo * +_PyStgInfo_FromType_NoState(PyObject *type) +{ + size_t type_basicsize =_Py_SIZE_ROUND_UP(PyType_Type.tp_basicsize, + ALIGNOF_MAX_ALIGN_T); + return (StgInfo *)((char *)type + type_basicsize); +} + // Initialize StgInfo on a newly created type static inline StgInfo * PyStgInfo_Init(ctypes_state *state, PyTypeObject *type) From 6e012ced6cc07a7502278e1849c5618d1ab54a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 23 May 2024 13:28:31 -0400 Subject: [PATCH 190/903] gh-119469: Fix _pyrepl reference leaks (#119470) --- Lib/test/test_pyrepl/test_interact.py | 6 +++- Lib/test/test_pyrepl/test_unix_console.py | 36 ++++++++++++++++------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index b3dc07c063f90e..10e34045bcf92d 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -27,9 +27,11 @@ def bar(self): a """) console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() with ( patch.object(InteractiveColoredConsole, "showsyntaxerror") as showsyntaxerror, patch.object(InteractiveColoredConsole, "runsource", wraps=console.runsource) as runsource, + contextlib.redirect_stdout(f), ): more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] self.assertFalse(more) @@ -71,7 +73,9 @@ def test_runsource_compiles_and_runs_code(self): def test_runsource_returns_false_for_successful_compilation(self): console = InteractiveColoredConsole() source = "print('Hello, world!')" - result = console.runsource(source) + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) self.assertFalse(result) @force_not_colorized diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index cec3ae033325ac..e1faa00caafc27 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -112,17 +112,18 @@ class TestConsole(TestCase): def test_simple_addition(self, _os_write): code = "12+34" events = code_to_events(code) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, b"1") _os_write.assert_any_call(ANY, b"2") _os_write.assert_any_call(ANY, b"+") _os_write.assert_any_call(ANY, b"3") _os_write.assert_any_call(ANY, b"4") + con.restore() def test_wrap(self, _os_write): code = "12+34" events = code_to_events(code) - _, _ = handle_events_narrow_unix_console(events) + _, con = handle_events_narrow_unix_console(events) _os_write.assert_any_call(ANY, b"1") _os_write.assert_any_call(ANY, b"2") _os_write.assert_any_call(ANY, b"+") @@ -130,6 +131,8 @@ def test_wrap(self, _os_write): _os_write.assert_any_call(ANY, b"\\") _os_write.assert_any_call(ANY, b"\n") _os_write.assert_any_call(ANY, b"4") + con.restore() + def test_cursor_left(self, _os_write): code = "1" @@ -137,8 +140,9 @@ def test_cursor_left(self, _os_write): code_to_events(code), [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], ) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + con.restore() def test_cursor_left_right(self, _os_write): code = "1" @@ -149,9 +153,10 @@ def test_cursor_left_right(self, _os_write): Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), ], ) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuf"] + b":1") + con.restore() def test_cursor_up(self, _os_write): code = "1\n2+3" @@ -159,8 +164,9 @@ def test_cursor_up(self, _os_write): code_to_events(code), [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], ) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + con.restore() def test_cursor_up_down(self, _os_write): code = "1\n2+3" @@ -171,9 +177,10 @@ def test_cursor_up_down(self, _os_write): Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), ], ) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cud"] + b":1") + con.restore() def test_cursor_back_write(self, _os_write): events = itertools.chain( @@ -181,10 +188,11 @@ def test_cursor_back_write(self, _os_write): [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], code_to_events("2"), ) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, b"1") _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") _os_write.assert_any_call(ANY, b"2") + con.restore() def test_multiline_function_move_up_short_terminal(self, _os_write): # fmt: off @@ -201,8 +209,9 @@ def test_multiline_function_move_up_short_terminal(self, _os_write): Event(evt="scroll", data=None), ], ) - _, _ = handle_events_short_unix_console(events) + _, con = handle_events_short_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + con.restore() def test_multiline_function_move_up_down_short_terminal(self, _os_write): # fmt: off @@ -221,9 +230,10 @@ def test_multiline_function_move_up_down_short_terminal(self, _os_write): Event(evt="scroll", data=None), ], ) - _, _ = handle_events_short_unix_console(events) + _, con = handle_events_short_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ind"] + b":") + con.restore() def test_resize_bigger_on_multiline_function(self, _os_write): # fmt: off @@ -246,7 +256,7 @@ def same_console(events): console.get_event = MagicMock(side_effect=events) return console - _, _ = handle_all_events( + _, con = handle_all_events( [Event(evt="resize", data=None)], prepare_reader=same_reader, prepare_console=same_console, @@ -258,6 +268,8 @@ def same_console(events): call(ANY, b"def f():"), ] ) + console.restore() + con.restore() def test_resize_smaller_on_multiline_function(self, _os_write): # fmt: off @@ -280,7 +292,7 @@ def same_console(events): console.get_event = MagicMock(side_effect=events) return console - _, _ = handle_all_events( + _, con = handle_all_events( [Event(evt="resize", data=None)], prepare_reader=same_reader, prepare_console=same_console, @@ -292,3 +304,5 @@ def same_console(events): call(ANY, b" foo"), ] ) + console.restore() + con.restore() From b30d30c747df2bf9f1614df8e76db2ffdb24fcd8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 23 May 2024 15:15:52 -0400 Subject: [PATCH 191/903] gh-117398: Statically Allocate the Datetime C-API (GH-119472) --- ...-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst | 3 + Modules/_datetimemodule.c | 118 ++++++++++++------ Tools/c-analyzer/cpython/globals-to-fix.tsv | 3 + 3 files changed, 89 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst diff --git a/Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst b/Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst new file mode 100644 index 00000000000000..ac595f1b7fc84c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst @@ -0,0 +1,3 @@ +Objects in the datetime C-API are now all statically allocated, which means +better memory safety, especially when the module is reloaded. This should be +transparent to users. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9a66f0358179b5..3c6d270b8d1331 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1178,6 +1178,8 @@ new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, return t; } +static PyDateTime_Delta * look_up_delta(int, int, int, PyTypeObject *); + /* Create a timedelta instance. Normalize the members iff normalize is * true. Passing false is a speed optimization, if you know for sure * that seconds and microseconds are already in their proper ranges. In any @@ -1198,6 +1200,12 @@ new_delta_ex(int days, int seconds, int microseconds, int normalize, if (check_delta_day_range(days) < 0) return NULL; + self = look_up_delta(days, seconds, microseconds, type); + if (self != NULL) { + return (PyObject *)self; + } + assert(!PyErr_Occurred()); + self = (PyDateTime_Delta *) (type->tp_alloc(type, 0)); if (self != NULL) { self->hashcode = -1; @@ -1219,6 +1227,8 @@ typedef struct PyObject *name; } PyDateTime_TimeZone; +static PyDateTime_TimeZone * look_up_timezone(PyObject *offset, PyObject *name); + /* Create new timezone instance checking offset range. This function does not check the name argument. Caller must assure that offset is a timedelta instance and name is either NULL @@ -1234,6 +1244,12 @@ create_timezone(PyObject *offset, PyObject *name) assert(PyDelta_Check(offset)); assert(name == NULL || PyUnicode_Check(name)); + self = look_up_timezone(offset, name); + if (self != NULL) { + return (PyObject *)self; + } + assert(!PyErr_Occurred()); + self = (PyDateTime_TimeZone *)(type->tp_alloc(type, 0)); if (self == NULL) { return NULL; @@ -2892,6 +2908,25 @@ static PyTypeObject PyDateTime_DeltaType = { 0, /* tp_free */ }; +// XXX Can we make this const? +static PyDateTime_Delta zero_delta = { + PyObject_HEAD_INIT(&PyDateTime_DeltaType) + /* Letting this be set lazily is a benign race. */ + .hashcode = -1, +}; + +static PyDateTime_Delta * +look_up_delta(int days, int seconds, int microseconds, PyTypeObject *type) +{ + if (days == 0 && seconds == 0 && microseconds == 0 + && type == zero_delta.ob_base.ob_type) + { + return &zero_delta; + } + return NULL; +} + + /* * PyDateTime_Date implementation. */ @@ -4184,6 +4219,23 @@ static PyTypeObject PyDateTime_TimeZoneType = { timezone_new, /* tp_new */ }; +// XXX Can we make this const? +static PyDateTime_TimeZone utc_timezone = { + PyObject_HEAD_INIT(&PyDateTime_TimeZoneType) + .offset = (PyObject *)&zero_delta, + .name = NULL, +}; + +static PyDateTime_TimeZone * +look_up_timezone(PyObject *offset, PyObject *name) +{ + if (offset == utc_timezone.offset && name == NULL) { + return &utc_timezone; + } + return NULL; +} + + /* * PyDateTime_Time implementation. */ @@ -6719,45 +6771,42 @@ static PyMethodDef module_methods[] = { {NULL, NULL} }; + +/* The C-API is process-global. This violates interpreter isolation + * due to the objects stored here. Thus each of those objects must + * be managed carefully. */ +// XXX Can we make this const? +static PyDateTime_CAPI capi = { + /* The classes must be readied before used here. + * That will happen the first time the module is loaded. + * They aren't safe to be shared between interpreters, + * but that's okay as long as the module is single-phase init. */ + .DateType = &PyDateTime_DateType, + .DateTimeType = &PyDateTime_DateTimeType, + .TimeType = &PyDateTime_TimeType, + .DeltaType = &PyDateTime_DeltaType, + .TZInfoType = &PyDateTime_TZInfoType, + + .TimeZone_UTC = (PyObject *)&utc_timezone, + + .Date_FromDate = new_date_ex, + .DateTime_FromDateAndTime = new_datetime_ex, + .Time_FromTime = new_time_ex, + .Delta_FromDelta = new_delta_ex, + .TimeZone_FromTimeZone = new_timezone, + .DateTime_FromTimestamp = datetime_fromtimestamp, + .Date_FromTimestamp = datetime_date_fromtimestamp_capi, + .DateTime_FromDateAndTimeAndFold = new_datetime_ex2, + .Time_FromTimeAndFold = new_time_ex2, +}; + /* Get a new C API by calling this function. * Clients get at C API via PyDateTime_IMPORT, defined in datetime.h. */ static inline PyDateTime_CAPI * get_datetime_capi(void) { - datetime_state *st = get_datetime_state(); - - PyDateTime_CAPI *capi = PyMem_Malloc(sizeof(PyDateTime_CAPI)); - if (capi == NULL) { - PyErr_NoMemory(); - return NULL; - } - capi->DateType = st->date_type; - capi->DateTimeType = st->datetime_type; - capi->TimeType = st->time_type; - capi->DeltaType = st->delta_type; - capi->TZInfoType = st->tzinfo_type; - capi->Date_FromDate = new_date_ex; - capi->DateTime_FromDateAndTime = new_datetime_ex; - capi->Time_FromTime = new_time_ex; - capi->Delta_FromDelta = new_delta_ex; - capi->TimeZone_FromTimeZone = new_timezone; - capi->DateTime_FromTimestamp = datetime_fromtimestamp; - capi->Date_FromTimestamp = datetime_date_fromtimestamp_capi; - capi->DateTime_FromDateAndTimeAndFold = new_datetime_ex2; - capi->Time_FromTimeAndFold = new_time_ex2; - // Make sure this function is called after utc has - // been initialized. - assert(st->utc != NULL); - capi->TimeZone_UTC = st->utc; // borrowed ref - return capi; -} - -static void -datetime_destructor(PyObject *op) -{ - void *ptr = PyCapsule_GetPointer(op, PyDateTime_CAPSULE_NAME); - PyMem_Free(ptr); + return &capi; } static int @@ -6955,8 +7004,7 @@ _datetime_exec(PyObject *module) if (capi == NULL) { goto error; } - PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, - datetime_destructor); + PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, NULL); if (capsule == NULL) { PyMem_Free(capi); goto error; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 8b6fe94e3afc52..711ae343a8d876 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -304,6 +304,9 @@ Python/crossinterp_exceptions.h - PyExc_InterpreterNotFoundError - ##----------------------- ## singletons +Modules/_datetimemodule.c - zero_delta - +Modules/_datetimemodule.c - utc_timezone - +Modules/_datetimemodule.c - capi - Objects/boolobject.c - _Py_FalseStruct - Objects/boolobject.c - _Py_TrueStruct - Objects/dictobject.c - empty_keys_struct - From be1dfccdf2c5c7671b8a549e969b8cf7d60d9936 Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Thu, 23 May 2024 16:59:35 -0400 Subject: [PATCH 192/903] gh-118727: Don't drop the GIL in `drop_gil()` unless the current thread holds it (#118745) `drop_gil()` assumes that its caller is attached, which means that the current thread holds the GIL if and only if the GIL is enabled, and the enabled-state of the GIL won't change. This isn't true, though, because `detach_thread()` calls `_PyEval_ReleaseLock()` after detaching and `_PyThreadState_DeleteCurrent()` calls it after removing the current thread from consideration for stop-the-world requests (effectively detaching it). Fix this by remembering whether or not a thread acquired the GIL when it last attached, in `PyThreadState._status.holds_gil`, and check this in `drop_gil()` instead of `gil->enabled`. This fixes a crash in `test_multiprocessing_pool_circular_import()`, so I've reenabled it. --- Include/cpython/pystate.h | 4 +- Include/internal/pycore_ceval.h | 7 +- .../test_importlib/test_threaded_import.py | 5 +- Python/ceval_gil.c | 96 +++++++++++-------- Python/pystate.c | 11 +-- 5 files changed, 68 insertions(+), 55 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 2df9ecd6d52084..ed3ee090ae53db 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -83,6 +83,8 @@ struct _ts { unsigned int bound_gilstate:1; /* Currently in use (maybe holds the GIL). */ unsigned int active:1; + /* Currently holds the GIL. */ + unsigned int holds_gil:1; /* various stages of finalization */ unsigned int finalizing:1; @@ -90,7 +92,7 @@ struct _ts { unsigned int finalized:1; /* padding to align to 4 bytes */ - unsigned int :24; + unsigned int :23; } _status; #ifdef Py_BUILD_CORE # define _PyThreadState_WHENCE_NOTSET -1 diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 48ad0678995904..bd3ba1225f2597 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -131,11 +131,10 @@ extern int _PyEval_ThreadsInitialized(void); extern void _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); -// Acquire the GIL and return 1. In free-threaded builds, this function may -// return 0 to indicate that the GIL was disabled and therefore not acquired. -extern int _PyEval_AcquireLock(PyThreadState *tstate); +extern void _PyEval_AcquireLock(PyThreadState *tstate); -extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *); +extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *, + int final_release); #ifdef Py_GIL_DISABLED // Returns 0 or 1 if the GIL for the given thread's interpreter is disabled or diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 3477112927a0b6..9af1e4d505c66e 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -17,7 +17,7 @@ from test.support import verbose from test.support.import_helper import forget, mock_register_at_fork from test.support.os_helper import (TESTFN, unlink, rmtree) -from test.support import script_helper, threading_helper, requires_gil_enabled +from test.support import script_helper, threading_helper threading_helper.requires_working_threading(module=True) @@ -248,9 +248,6 @@ def test_concurrent_futures_circular_import(self): 'partial', 'cfimport.py') script_helper.assert_python_ok(fn) - # gh-118727 and gh-118729: pool_in_threads.py may crash in free-threaded - # builds, which can hang the Tsan test so temporarily skip it for now. - @requires_gil_enabled("gh-118727: test may crash in free-threaded builds") def test_multiprocessing_pool_circular_import(self): # Regression test for bpo-41567 fn = os.path.join(os.path.dirname(__file__), diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 7a54c185303cd1..5617504a495686 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -205,32 +205,39 @@ static void recreate_gil(struct _gil_runtime_state *gil) } #endif -static void -drop_gil_impl(struct _gil_runtime_state *gil) +static inline void +drop_gil_impl(PyThreadState *tstate, struct _gil_runtime_state *gil) { MUTEX_LOCK(gil->mutex); _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); _Py_atomic_store_int_relaxed(&gil->locked, 0); + if (tstate != NULL) { + tstate->_status.holds_gil = 0; + } COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); } static void -drop_gil(PyInterpreterState *interp, PyThreadState *tstate) +drop_gil(PyInterpreterState *interp, PyThreadState *tstate, int final_release) { struct _ceval_state *ceval = &interp->ceval; - /* If tstate is NULL, the caller is indicating that we're releasing + /* If final_release is true, the caller is indicating that we're releasing the GIL for the last time in this thread. This is particularly relevant when the current thread state is finalizing or its interpreter is finalizing (either may be in an inconsistent state). In that case the current thread will definitely never try to acquire the GIL again. */ // XXX It may be more correct to check tstate->_status.finalizing. - // XXX assert(tstate == NULL || !tstate->_status.cleared); + // XXX assert(final_release || !tstate->_status.cleared); + assert(final_release || tstate != NULL); struct _gil_runtime_state *gil = ceval->gil; #ifdef Py_GIL_DISABLED - if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { + // Check if we have the GIL before dropping it. tstate will be NULL if + // take_gil() detected that this thread has been destroyed, in which case + // we know we have the GIL. + if (tstate != NULL && !tstate->_status.holds_gil) { return; } #endif @@ -238,26 +245,23 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) Py_FatalError("drop_gil: GIL is not locked"); } - /* tstate is allowed to be NULL (early interpreter init) */ - if (tstate != NULL) { + if (!final_release) { /* Sub-interpreter support: threads might have been switched under our feet using PyThreadState_Swap(). Fix the GIL last holder variable so that our heuristics work. */ _Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate); } - drop_gil_impl(gil); + drop_gil_impl(tstate, gil); #ifdef FORCE_SWITCHING - /* We check tstate first in case we might be releasing the GIL for - the last time in this thread. In that case there's a possible - race with tstate->interp getting deleted after gil->mutex is - unlocked and before the following code runs, leading to a crash. - We can use (tstate == NULL) to indicate the thread is done with - the GIL, and that's the only time we might delete the - interpreter, so checking tstate first prevents the crash. - See https://github.com/python/cpython/issues/104341. */ - if (tstate != NULL && + /* We might be releasing the GIL for the last time in this thread. In that + case there's a possible race with tstate->interp getting deleted after + gil->mutex is unlocked and before the following code runs, leading to a + crash. We can use final_release to indicate the thread is done with the + GIL, and that's the only time we might delete the interpreter. See + https://github.com/python/cpython/issues/104341. */ + if (!final_release && _Py_eval_breaker_bit_is_set(tstate, _PY_GIL_DROP_REQUEST_BIT)) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ @@ -284,7 +288,7 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) tstate must be non-NULL. Returns 1 if the GIL was acquired, or 0 if not. */ -static int +static void take_gil(PyThreadState *tstate) { int err = errno; @@ -309,7 +313,7 @@ take_gil(PyThreadState *tstate) struct _gil_runtime_state *gil = interp->ceval.gil; #ifdef Py_GIL_DISABLED if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { - return 0; + return; } #endif @@ -358,10 +362,10 @@ take_gil(PyThreadState *tstate) if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { // Another thread disabled the GIL between our check above and // now. Don't take the GIL, signal any other waiting threads, and - // return 0. + // return. COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); - return 0; + return; } #endif @@ -393,20 +397,21 @@ take_gil(PyThreadState *tstate) in take_gil() while the main thread called wait_for_thread_shutdown() from Py_Finalize(). */ MUTEX_UNLOCK(gil->mutex); - /* Passing NULL to drop_gil() indicates that this thread is about to - terminate and will never hold the GIL again. */ - drop_gil(interp, NULL); + /* tstate could be a dangling pointer, so don't pass it to + drop_gil(). */ + drop_gil(interp, NULL, 1); PyThread_exit_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); + tstate->_status.holds_gil = 1; _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); update_eval_breaker_for_thread(interp, tstate); MUTEX_UNLOCK(gil->mutex); errno = err; - return 1; + return; } void _PyEval_SetSwitchInterval(unsigned long microseconds) @@ -451,10 +456,17 @@ PyEval_ThreadsInitialized(void) static inline int current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) { - if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) != tstate) { - return 0; - } - return _Py_atomic_load_int_relaxed(&gil->locked); + int holds_gil = tstate->_status.holds_gil; + + // holds_gil is the source of truth; check that last_holder and gil->locked + // are consistent with it. + int locked = _Py_atomic_load_int_relaxed(&gil->locked); + int is_last_holder = + ((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate; + assert(!holds_gil || locked); + assert(!holds_gil || is_last_holder); + + return holds_gil; } #endif @@ -563,23 +575,24 @@ PyEval_ReleaseLock(void) /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ - drop_gil(tstate->interp, tstate); + drop_gil(tstate->interp, tstate, 0); } -int +void _PyEval_AcquireLock(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); - return take_gil(tstate); + take_gil(tstate); } void -_PyEval_ReleaseLock(PyInterpreterState *interp, PyThreadState *tstate) +_PyEval_ReleaseLock(PyInterpreterState *interp, + PyThreadState *tstate, + int final_release) { - /* If tstate is NULL then we do not expect the current thread - to acquire the GIL ever again. */ - assert(tstate == NULL || tstate->interp == interp); - drop_gil(interp, tstate); + assert(tstate != NULL); + assert(tstate->interp == interp); + drop_gil(interp, tstate, final_release); } void @@ -1136,7 +1149,12 @@ _PyEval_DisableGIL(PyThreadState *tstate) // // Drop the GIL, which will wake up any threads waiting in take_gil() // and let them resume execution without the GIL. - drop_gil_impl(gil); + drop_gil_impl(tstate, gil); + + // If another thread asked us to drop the GIL, they should be + // free-threading by now. Remove any such request so we have a clean + // slate if/when the GIL is enabled again. + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); return 1; } return 0; diff --git a/Python/pystate.c b/Python/pystate.c index 0832b37c278c76..1ea1ad982a0ec9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1843,7 +1843,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) #endif current_fast_clear(tstate->interp->runtime); tstate_delete_common(tstate); - _PyEval_ReleaseLock(tstate->interp, NULL); + _PyEval_ReleaseLock(tstate->interp, tstate, 1); free_threadstate((_PyThreadStateImpl *)tstate); } @@ -2068,7 +2068,7 @@ _PyThreadState_Attach(PyThreadState *tstate) while (1) { - int acquired_gil = _PyEval_AcquireLock(tstate); + _PyEval_AcquireLock(tstate); // XXX assert(tstate_is_alive(tstate)); current_fast_set(&_PyRuntime, tstate); @@ -2079,20 +2079,17 @@ _PyThreadState_Attach(PyThreadState *tstate) } #ifdef Py_GIL_DISABLED - if (_PyEval_IsGILEnabled(tstate) != acquired_gil) { + if (_PyEval_IsGILEnabled(tstate) && !tstate->_status.holds_gil) { // The GIL was enabled between our call to _PyEval_AcquireLock() // and when we attached (the GIL can't go from enabled to disabled // here because only a thread holding the GIL can disable // it). Detach and try again. - assert(!acquired_gil); tstate_set_detached(tstate, _Py_THREAD_DETACHED); tstate_deactivate(tstate); current_fast_clear(&_PyRuntime); continue; } _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); -#else - (void)acquired_gil; #endif break; } @@ -2123,7 +2120,7 @@ detach_thread(PyThreadState *tstate, int detached_state) tstate_deactivate(tstate); tstate_set_detached(tstate, detached_state); current_fast_clear(&_PyRuntime); - _PyEval_ReleaseLock(tstate->interp, tstate); + _PyEval_ReleaseLock(tstate->interp, tstate, 0); } void From e94dbe4ed83460f18bd72563c5f09f6cdc71f604 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 23 May 2024 23:26:09 +0200 Subject: [PATCH 193/903] gh-119461: Fix ThreadedVSOCKSocketStreamTest (#119465) Add socket.VMADDR_CID_LOCAL constant. Fix ThreadedVSOCKSocketStreamTest: if get_cid() returns the host address or the "any" address, use the local communication address (loopback): VMADDR_CID_LOCAL. On Linux 6.9, apparently, the /dev/vsock device is now available but get_cid() returns VMADDR_CID_ANY (-1). --- Lib/test/test_socket.py | 10 ++++++---- .../2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst | 1 + Modules/socketmodule.c | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 0c4b3bb2ad4d81..ce0f64b43ed49f 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -160,8 +160,8 @@ def _have_socket_qipcrtr(): def _have_socket_vsock(): """Check whether AF_VSOCK sockets are supported on this host.""" - ret = get_cid() is not None - return ret + cid = get_cid() + return (cid is not None) def _have_socket_bluetooth(): @@ -520,8 +520,6 @@ def clientTearDown(self): @unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') -@unittest.skipUnless(get_cid() != 2, - "This test can only be run on a virtual guest.") class ThreadedVSOCKSocketStreamTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): @@ -543,6 +541,9 @@ def clientSetUp(self): self.cli = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) self.addCleanup(self.cli.close) cid = get_cid() + if cid in (socket.VMADDR_CID_HOST, socket.VMADDR_CID_ANY): + # gh-119461: Use the local communication address (loopback) + cid = socket.VMADDR_CID_LOCAL self.cli.connect((cid, VSOCKPORT)) def testStream(self): @@ -2515,6 +2516,7 @@ def testVSOCKConstants(self): socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE socket.VMADDR_CID_ANY socket.VMADDR_PORT_ANY + socket.VMADDR_CID_LOCAL socket.VMADDR_CID_HOST socket.VM_SOCKETS_INVALID_VERSION socket.IOCTL_VM_SOCKETS_GET_LOCAL_CID diff --git a/Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst b/Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst new file mode 100644 index 00000000000000..48e18f42b5556a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst @@ -0,0 +1 @@ +Add ``socket.VMADDR_CID_LOCAL`` constant. Patch by Victor Stinner. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index daec560ddfcac7..cb7dc25e23fb3d 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7596,6 +7596,7 @@ socket_exec(PyObject *m) ADD_INT_CONST(m, "SO_VM_SOCKETS_BUFFER_MAX_SIZE", 2); ADD_INT_CONST(m, "VMADDR_CID_ANY", 0xffffffff); ADD_INT_CONST(m, "VMADDR_PORT_ANY", 0xffffffff); + ADD_INT_CONST(m, "VMADDR_CID_LOCAL", 1); ADD_INT_CONST(m, "VMADDR_CID_HOST", 2); ADD_INT_CONST(m, "VM_SOCKETS_INVALID_VERSION", 0xffffffff); ADD_INT_CONST(m, "IOCTL_VM_SOCKETS_GET_LOCAL_CID", _IO(7, 0xb9)); From ffa24aab107b5bc3c6ad31a6a245c226bf24b208 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 24 May 2024 00:11:45 +0200 Subject: [PATCH 194/903] Clarify base64.a85encode docs: *wrapcols* doesn't count the newline (GH-119409) --- Doc/library/base64.rst | 2 +- Lib/base64.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index cec9a6cef4bf7d..834ab2536e6c14 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -193,7 +193,7 @@ The modern interface provides: *wrapcol* controls whether the output should have newline (``b'\n'``) characters added to it. If this is non-zero, each output line will be - at most this many characters long. + at most this many characters long, excluding the trailing newline. *pad* controls whether the input is padded to a multiple of 4 before encoding. Note that the ``btoa`` implementation always pads. diff --git a/Lib/base64.py b/Lib/base64.py index 25164d1a1df4fc..5a7e790a193380 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -332,7 +332,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): wrapcol controls whether the output should have newline (b'\\n') characters added to it. If this is non-zero, each output line will be at most this - many characters long. + many characters long, excluding the trailing newline. pad controls whether the input is padded to a multiple of 4 before encoding. Note that the btoa implementation always pads. From 0867bce45768454ee31bee95ca33fdc2c9d8b0fa Mon Sep 17 00:00:00 2001 From: Carlos Meza Date: Thu, 23 May 2024 19:04:12 -0700 Subject: [PATCH 195/903] gh-119317: findall instead of traverse for docutils nodes (#119319) --- Doc/tools/extensions/glossary_search.py | 4 ++-- Doc/tools/extensions/pyspecific.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/tools/extensions/glossary_search.py b/Doc/tools/extensions/glossary_search.py index 232782093926f6..7c93b1e4990603 100644 --- a/Doc/tools/extensions/glossary_search.py +++ b/Doc/tools/extensions/glossary_search.py @@ -25,8 +25,8 @@ def process_glossary_nodes(app, doctree, fromdocname): terms = {} - for node in doctree.traverse(glossary): - for glossary_item in node.traverse(definition_list_item): + for node in doctree.findall(glossary): + for glossary_item in node.findall(definition_list_item): term = glossary_item[0].astext().lower() definition = glossary_item[1] diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 44db77af5d24d3..8b592d4b4adcea 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -604,7 +604,7 @@ def parse_monitoring_event(env, sig, signode): def process_audit_events(app, doctree, fromdocname): - for node in doctree.traverse(audit_event_list): + for node in doctree.findall(audit_event_list): break else: return @@ -663,7 +663,7 @@ def process_audit_events(app, doctree, fromdocname): body += row - for node in doctree.traverse(audit_event_list): + for node in doctree.findall(audit_event_list): node.replace_self(table) From b48a3dbff4d70e72797e67b46276564fc63ddb89 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 23 May 2024 23:13:41 -0400 Subject: [PATCH 196/903] GH-113464: Run the JIT interpreter before any other JIT CI (GH-119466) --- .github/workflows/jit.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index b7938a177c856f..8c760a81d52662 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -26,8 +26,22 @@ concurrency: cancel-in-progress: true jobs: + interpreter: + name: Interpreter (Debug) + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - name: Build tier two interpreter + run: | + ./configure --enable-experimental-jit=interpreter --with-pydebug + make all --jobs 4 + - name: Test tier two interpreter + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 jit: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + needs: interpreter runs-on: ${{ matrix.runner }} timeout-minutes: 90 strategy: @@ -153,6 +167,7 @@ jobs: jit-with-disabled-gil: name: Free-Threaded (Debug) + needs: interpreter runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 92fab3356f4c61d4c73606e4fae705c6d8f6213b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 24 May 2024 14:31:40 +0200 Subject: [PATCH 197/903] gh-69214: Fix fcntl.ioctl() request type (#119498) Use an 'unsigned long' instead of an 'unsigned int' for the request parameter of fcntl.ioctl() to support requests larger than UINT_MAX. --- .../2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst | 3 +++ Modules/clinic/fcntlmodule.c.h | 11 ++++++----- Modules/fcntlmodule.c | 6 +++--- 3 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst diff --git a/Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst b/Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst new file mode 100644 index 00000000000000..8c3a36c9f56475 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst @@ -0,0 +1,3 @@ +Fix ``fcntl.ioctl()`` *request* parameter: use an ``unsigned long`` instead of +an ``unsigned int`` for the *request* parameter of :func:`fcntl.ioctl` to +support requests larger than ``UINT_MAX``. Patch by Victor Stinner. diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index d4846ddf8df7e4..53b139e09afdf1 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -96,7 +96,7 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, {"ioctl", (PyCFunction)(void(*)(void))fcntl_ioctl, METH_FASTCALL, fcntl_ioctl__doc__}, static PyObject * -fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, +fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *ob_arg, int mutate_arg); static PyObject * @@ -104,7 +104,7 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int fd; - unsigned int code; + unsigned long code; PyObject *ob_arg = NULL; int mutate_arg = 1; @@ -120,10 +120,11 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (fd < 0) { goto exit; } - code = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (code == (unsigned int)-1 && PyErr_Occurred()) { + if (!PyLong_Check(args[1])) { + PyErr_Format(PyExc_TypeError, "ioctl() argument 2 must be int, not %T", args[1]); goto exit; } + code = PyLong_AsUnsignedLongMask(args[1]); if (nargs < 3) { goto skip_optional; } @@ -263,4 +264,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=26793691ab1c75ba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=45a56f53fd17ff3c input=a9049054013a1b77]*/ diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index b6eeec2c66f6e5..873bdf2ac0657a 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -112,7 +112,7 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) fcntl.ioctl fd: fildes - request as code: unsigned_int(bitwise=True) + request as code: unsigned_long(bitwise=True) arg as ob_arg: object(c_default='NULL') = 0 mutate_flag as mutate_arg: bool = True / @@ -148,9 +148,9 @@ code. [clinic start generated code]*/ static PyObject * -fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, +fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *ob_arg, int mutate_arg) -/*[clinic end generated code: output=7f7f5840c65991be input=967b4a4cbeceb0a8]*/ +/*[clinic end generated code: output=3d8eb6828666cea1 input=cee70f6a27311e58]*/ { #define IOCTL_BUFSZ 1024 /* We use the unsigned non-checked 'I' format for the 'code' parameter From bf5b6467f8cc06759f3396ab1a8ad64fe7d1db2e Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Sat, 25 May 2024 00:29:19 +1000 Subject: [PATCH 198/903] GH-119496: accept UTF-8 BOM in .pth files (GH-119503) `Out-File -Encoding utf8` and similar commands in Windows Powershell 5.1 emit UTF-8 with a BOM marker, which the regular `utf-8` codec decodes incorrectly. `utf-8-sig` accepts a BOM, but also works correctly without one. This change also makes .pth files match the way Python source files are handled. Co-authored-by: Inada Naoki --- Lib/site.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index f1a6d9cf66fdc3..7eace190f5ab21 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -185,7 +185,9 @@ def addpackage(sitedir, name, known_paths): return try: - pth_content = pth_content.decode() + # Accept BOM markers in .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM) + pth_content = pth_content.decode("utf-8-sig") except UnicodeDecodeError: # Fallback to locale encoding for backward compatibility. # We will deprecate this fallback in the future. From f0ed1863bd7a0b9d021fb59e156663a7ec553f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Germ=C3=A1n=20M=C3=A9ndez=20Bravo?= Date: Fri, 24 May 2024 10:39:06 -0700 Subject: [PATCH 199/903] gh-112075: Fix dict thread safety issues (#119288) Fix dict thread safety issues --- Objects/dictobject.c | 66 +++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 6e1c3b93fd391b..a1ee32b7099f91 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -154,6 +154,11 @@ ASSERT_DICT_LOCKED(PyObject *op) _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); } #define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op)) +#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + ASSERT_DICT_LOCKED(op); \ + } + #define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp) #define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp) #define LOAD_INDEX(keys, size, idx) _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)keys->dk_indices)[idx]); @@ -221,6 +226,7 @@ static inline void split_keys_entry_added(PyDictKeysObject *keys) #else /* Py_GIL_DISABLED */ #define ASSERT_DICT_LOCKED(op) +#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) #define LOCK_KEYS(keys) #define UNLOCK_KEYS(keys) #define ASSERT_KEYS_LOCKED(keys) @@ -473,7 +479,7 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_IMMORTAL_REFCNT) { return; } - assert(dk->dk_refcnt > 0); + assert(FT_ATOMIC_LOAD_SSIZE(dk->dk_refcnt) > 0); #ifdef Py_REF_DEBUG _Py_DecRefTotal(_PyThreadState_GET()); #endif @@ -670,6 +676,8 @@ dump_entries(PyDictKeysObject *dk) int _PyDict_CheckConsistency(PyObject *op, int check_content) { + ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op); + #define CHECK(expr) \ do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0) @@ -1580,6 +1588,8 @@ _PyDict_MaybeUntrack(PyObject *op) PyObject *value; Py_ssize_t i, numentries; + ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op); + if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op)) return; @@ -1722,13 +1732,14 @@ static void insert_split_value(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix) { assert(PyUnicode_CheckExact(key)); + ASSERT_DICT_LOCKED(mp); MAINTAIN_TRACKING(mp, key, value); PyObject *old_value = mp->ma_values->values[ix]; if (old_value == NULL) { uint64_t new_version = _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); - mp->ma_used++; + STORE_USED(mp, mp->ma_used + 1); mp->ma_version_tag = new_version; } else { @@ -1792,7 +1803,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, goto Fail; } mp->ma_version_tag = new_version; - mp->ma_used++; + STORE_USED(mp, mp->ma_used + 1); ASSERT_CONSISTENT(mp); return 0; } @@ -1861,7 +1872,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, ep->me_hash = hash; STORE_VALUE(ep, value); } - FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, FT_ATOMIC_LOAD_SSIZE_RELAXED(mp->ma_used) + 1); + STORE_USED(mp, mp->ma_used + 1); mp->ma_version_tag = new_version; newkeys->dk_usable--; newkeys->dk_nentries++; @@ -1870,11 +1881,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, // the case where we're inserting from the non-owner thread. We don't use // set_keys here because the transition from empty to non-empty is safe // as the empty keys will never be freed. -#ifdef Py_GIL_DISABLED - _Py_atomic_store_ptr_release(&mp->ma_keys, newkeys); -#else - mp->ma_keys = newkeys; -#endif + FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_keys, newkeys); return 0; } @@ -2580,7 +2587,7 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); - FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, FT_ATOMIC_LOAD_SSIZE(mp->ma_used) - 1); + STORE_USED(mp, mp->ma_used - 1); mp->ma_version_tag = new_version; if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); @@ -2752,7 +2759,7 @@ clear_lock_held(PyObject *op) // We don't inc ref empty keys because they're immortal ensure_shared_on_resize(mp); mp->ma_version_tag = new_version; - mp->ma_used = 0; + STORE_USED(mp, 0); if (oldvalues == NULL) { set_keys(mp, Py_EMPTY_KEYS); assert(oldkeys->dk_refcnt == 1); @@ -3191,6 +3198,8 @@ dict_repr_lock_held(PyObject *self) _PyUnicodeWriter writer; int first; + ASSERT_DICT_LOCKED(mp); + i = Py_ReprEnter((PyObject *)mp); if (i != 0) { return i > 0 ? PyUnicode_FromString("{...}") : NULL; @@ -3279,8 +3288,7 @@ dict_repr(PyObject *self) static Py_ssize_t dict_length(PyObject *self) { - PyDictObject *mp = (PyDictObject *)self; - return _Py_atomic_load_ssize_relaxed(&mp->ma_used); + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)self)->ma_used); } static PyObject * @@ -3672,6 +3680,9 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) static int dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *other, int override) { + ASSERT_DICT_LOCKED(mp); + ASSERT_DICT_LOCKED(other); + if (other == mp || other->ma_used == 0) /* a.update(a) or a.update({}); nothing to do */ return 0; @@ -3699,7 +3710,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe ensure_shared_on_resize(mp); dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; - mp->ma_used = other->ma_used; + STORE_USED(mp, other->ma_used); mp->ma_version_tag = new_version; ASSERT_CONSISTENT(mp); @@ -4034,7 +4045,7 @@ PyDict_Size(PyObject *mp) PyErr_BadInternalCall(); return -1; } - return ((PyDictObject *)mp)->ma_used; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)mp)->ma_used); } /* Return 1 if dicts equal, 0 if not, -1 if error. @@ -4291,7 +4302,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } MAINTAIN_TRACKING(mp, key, value); - mp->ma_used++; + STORE_USED(mp, mp->ma_used + 1); mp->ma_version_tag = new_version; assert(mp->ma_keys->dk_usable >= 0); ASSERT_CONSISTENT(mp); @@ -4413,6 +4424,8 @@ dict_popitem_impl(PyDictObject *self) uint64_t new_version; PyInterpreterState *interp = _PyInterpreterState_GET(); + ASSERT_DICT_LOCKED(self); + /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which * could empty the dict, so if we checked the size first and that @@ -4952,19 +4965,21 @@ typedef struct { static PyObject * dictiter_new(PyDictObject *dict, PyTypeObject *itertype) { + Py_ssize_t used; dictiterobject *di; di = PyObject_GC_New(dictiterobject, itertype); if (di == NULL) { return NULL; } di->di_dict = (PyDictObject*)Py_NewRef(dict); - di->di_used = dict->ma_used; - di->len = dict->ma_used; + used = FT_ATOMIC_LOAD_SSIZE_RELAXED(dict->ma_used); + di->di_used = used; + di->len = used; if (itertype == &PyDictRevIterKey_Type || itertype == &PyDictRevIterItem_Type || itertype == &PyDictRevIterValue_Type) { if (_PyDict_HasSplitTable(dict)) { - di->di_pos = dict->ma_used - 1; + di->di_pos = used - 1; } else { di->di_pos = load_keys_nentries(dict) - 1; @@ -5013,8 +5028,8 @@ dictiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { dictiterobject *di = (dictiterobject *)self; Py_ssize_t len = 0; - if (di->di_dict != NULL && di->di_used == di->di_dict->ma_used) - len = di->len; + if (di->di_dict != NULL && di->di_used == FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_dict->ma_used)) + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->len); return PyLong_FromSize_t(len); } @@ -5297,6 +5312,7 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, Py_ssize_t i; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -5811,7 +5827,7 @@ dictview_len(PyObject *self) _PyDictViewObject *dv = (_PyDictViewObject *)self; Py_ssize_t len = 0; if (dv->dv_dict != NULL) - len = dv->dv_dict->ma_used; + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(dv->dv_dict->ma_used); return len; } @@ -6820,7 +6836,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, _PyDictValues_AddToInsertionOrder(values, ix); if (dict) { assert(dict->ma_values == values); - dict->ma_used++; + STORE_USED(dict, dict->ma_used + 1); } } else { @@ -6828,7 +6844,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, delete_index_from_values(values, ix); if (dict) { assert(dict->ma_values == values); - dict->ma_used--; + STORE_USED(dict, dict->ma_used - 1); } } Py_DECREF(old_value); @@ -7039,7 +7055,7 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) if (dict == NULL) { return 1; } - return ((PyDictObject *)dict)->ma_used == 0; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)dict)->ma_used) == 0; } int From 96b392df303b2cfaea823afcb462c0b455704ce8 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Fri, 24 May 2024 20:04:17 +0200 Subject: [PATCH 200/903] gh-118263: Add additional arguments to path_t (Argument Clinic type) in posixmodule (GH-118355) --- Lib/ntpath.py | 42 +- Lib/posixpath.py | 41 +- Lib/test/test_ntpath.py | 4 + Lib/test/test_posixpath.py | 13 +- ...-04-28-19-51-00.gh-issue-118263.Gaap3S.rst | 1 + Modules/clinic/posixmodule.c.h | 227 ++++++---- Modules/posixmodule.c | 420 ++++++++++-------- 7 files changed, 389 insertions(+), 359 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 8d972cd1d0eb72..83e2d3b865757c 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -168,19 +168,12 @@ def splitdrive(p): try: - from nt import _path_splitroot_ex + from nt import _path_splitroot_ex as splitroot except ImportError: def splitroot(p): - """Split a pathname into drive, root and tail. The drive is defined - exactly as in splitdrive(). On Windows, the root may be a single path - separator or an empty string. The tail contains anything after the root. - For example: - - splitroot('//server/share/') == ('//server/share', '/', '') - splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') - splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') - splitroot('Windows/notepad') == ('', '', 'Windows/notepad') - """ + """Split a pathname into drive, root and tail. + + The tail contains anything after the root.""" p = os.fspath(p) if isinstance(p, bytes): sep = b'\\' @@ -220,23 +213,6 @@ def splitroot(p): else: # Relative path, e.g. Windows return empty, empty, p -else: - def splitroot(p): - """Split a pathname into drive, root and tail. The drive is defined - exactly as in splitdrive(). On Windows, the root may be a single path - separator or an empty string. The tail contains anything after the root. - For example: - - splitroot('//server/share/') == ('//server/share', '/', '') - splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') - splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') - splitroot('Windows/notepad') == ('', '', 'Windows/notepad') - """ - p = os.fspath(p) - if isinstance(p, bytes): - drive, root, tail = _path_splitroot_ex(os.fsdecode(p)) - return os.fsencode(drive), os.fsencode(root), os.fsencode(tail) - return _path_splitroot_ex(p) # Split a path in head (everything up to the last '/') and tail (the @@ -538,7 +514,7 @@ def expandvars(path): # Previously, this function also truncated pathnames to 8+3 format, # but as this module is called "ntpath", that's obviously wrong! try: - from nt import _path_normpath + from nt import _path_normpath as normpath except ImportError: def normpath(path): @@ -577,14 +553,6 @@ def normpath(path): comps.append(curdir) return prefix + sep.join(comps) -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - def _abspath_fallback(path): """Return the absolute version of a path as a fallback function in case diff --git a/Lib/posixpath.py b/Lib/posixpath.py index c04c628de55ee2..47b2aa572e5c65 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -135,18 +135,12 @@ def splitdrive(p): try: - from posix import _path_splitroot_ex + from posix import _path_splitroot_ex as splitroot except ImportError: def splitroot(p): - """Split a pathname into drive, root and tail. On Posix, drive is always - empty; the root may be empty, a single slash, or two slashes. The tail - contains anything after the root. For example: - - splitroot('foo/bar') == ('', '', 'foo/bar') - splitroot('/foo/bar') == ('', '/', 'foo/bar') - splitroot('//foo/bar') == ('', '//', 'foo/bar') - splitroot('///foo/bar') == ('', '/', '//foo/bar') - """ + """Split a pathname into drive, root and tail. + + The tail contains anything after the root.""" p = os.fspath(p) if isinstance(p, bytes): sep = b'/' @@ -164,23 +158,6 @@ def splitroot(p): # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 return empty, p[:2], p[2:] -else: - def splitroot(p): - """Split a pathname into drive, root and tail. On Posix, drive is always - empty; the root may be empty, a single slash, or two slashes. The tail - contains anything after the root. For example: - - splitroot('foo/bar') == ('', '', 'foo/bar') - splitroot('/foo/bar') == ('', '/', 'foo/bar') - splitroot('//foo/bar') == ('', '//', 'foo/bar') - splitroot('///foo/bar') == ('', '/', '//foo/bar') - """ - p = os.fspath(p) - if isinstance(p, bytes): - # Optimisation: the drive is always empty - _, root, tail = _path_splitroot_ex(os.fsdecode(p)) - return b'', os.fsencode(root), os.fsencode(tail) - return _path_splitroot_ex(p) # Return the tail (basename) part of a path, same as split(path)[1]. @@ -363,7 +340,7 @@ def expandvars(path): # if it contains symbolic links! try: - from posix import _path_normpath + from posix import _path_normpath as normpath except ImportError: def normpath(path): @@ -394,14 +371,6 @@ def normpath(path): path = initial_slashes + sep.join(comps) return path or dot -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - def abspath(path): """Return an absolute path.""" diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 9aa116682f7480..64cbfaaaaa0690 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1129,6 +1129,10 @@ def test_fast_paths_in_use(self): # There are fast paths of these functions implemented in posixmodule.c. # Confirm that they are being used, and not the Python fallbacks in # genericpath.py. + self.assertTrue(os.path.splitroot is nt._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is nt._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) self.assertTrue(os.path.isdir is nt._path_isdir) self.assertFalse(inspect.isfunction(os.path.isdir)) self.assertTrue(os.path.isfile is nt._path_isfile) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 238baed5efa264..57a24e9c70d5e5 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,3 +1,4 @@ +import inspect import os import posixpath import sys @@ -5,7 +6,7 @@ from posixpath import realpath, abspath, dirname, basename from test import test_genericpath from test.support import import_helper -from test.support import os_helper +from test.support import cpython_only, os_helper from test.support.os_helper import FakePath from unittest import mock @@ -283,6 +284,16 @@ def fake_lstat(path): def test_isjunction(self): self.assertFalse(posixpath.isjunction(ABSTFN)) + @unittest.skipIf(sys.platform == 'win32', "Fast paths are not for win32") + @cpython_only + def test_fast_paths_in_use(self): + # There are fast paths of these functions implemented in posixmodule.c. + # Confirm that they are being used, and not the Python fallbacks + self.assertTrue(os.path.splitroot is posix._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is posix._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) + def test_expanduser(self): self.assertEqual(posixpath.expanduser("foo"), "foo") self.assertEqual(posixpath.expanduser(b"foo"), b"foo") diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst new file mode 100644 index 00000000000000..165a1ba69a811b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst @@ -0,0 +1 @@ +Speed up :func:`os.path.splitroot` & :func:`os.path.normpath` with a direct C call. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 5ec5635bae3f41..c7a447b455c594 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -72,7 +72,7 @@ os_stat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("stat", "path", 0, 1); + path_t path = PATH_T_INITIALIZE_P("stat", "path", 0, 0, 0, 1); int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; @@ -154,7 +154,7 @@ os_lstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("lstat", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lstat", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -250,7 +250,7 @@ os_access(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("access", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("access", "path", 0, 0, 0, 0); int mode; int dir_fd = DEFAULT_DIR_FD; int effective_ids = 0; @@ -409,7 +409,7 @@ os_chdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("chdir", "path", 0, PATH_HAVE_FCHDIR); + path_t path = PATH_T_INITIALIZE_P("chdir", "path", 0, 0, 0, PATH_HAVE_FCHDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -560,7 +560,7 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("chmod", "path", 0, PATH_HAVE_FCHMOD); + path_t path = PATH_T_INITIALIZE_P("chmod", "path", 0, 0, 0, PATH_HAVE_FCHMOD); int mode; int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = CHMOD_DEFAULT_FOLLOW_SYMLINKS; @@ -725,7 +725,7 @@ os_lchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("lchmod", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchmod", "path", 0, 0, 0, 0); int mode; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -802,7 +802,7 @@ os_chflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("chflags", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("chflags", "path", 0, 0, 0, 0); unsigned long flags; int follow_symlinks = 1; @@ -884,7 +884,7 @@ os_lchflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("lchflags", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchflags", "path", 0, 0, 0, 0); unsigned long flags; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -954,7 +954,7 @@ os_chroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("chroot", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("chroot", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1190,7 +1190,7 @@ os_chown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("chown", "path", 0, PATH_HAVE_FCHOWN); + path_t path = PATH_T_INITIALIZE_P("chown", "path", 0, 0, 0, PATH_HAVE_FCHOWN); uid_t uid; gid_t gid; int dir_fd = DEFAULT_DIR_FD; @@ -1355,7 +1355,7 @@ os_lchown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[3]; - path_t path = PATH_T_INITIALIZE("lchown", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchown", "path", 0, 0, 0, 0); uid_t uid; gid_t gid; @@ -1476,8 +1476,8 @@ os_link(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("link", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("link", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("link", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("link", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; @@ -1583,7 +1583,7 @@ os_listdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("listdir", "path", 1, PATH_HAVE_FDOPENDIR); + path_t path = PATH_T_INITIALIZE_P("listdir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { @@ -1699,7 +1699,7 @@ os_listmounts(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t volume = PATH_T_INITIALIZE("listmounts", "volume", 0, 0); + path_t volume = PATH_T_INITIALIZE_P("listmounts", "volume", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1763,7 +1763,7 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_isdevdrive", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_path_isdevdrive", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1800,7 +1800,7 @@ static PyObject * os__getfullpathname(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfullpathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getfullpathname", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1834,7 +1834,7 @@ static PyObject * os__getfinalpathname(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfinalpathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getfinalpathname", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1868,7 +1868,7 @@ static PyObject * os__findfirstfile(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_findfirstfile", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_findfirstfile", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1928,7 +1928,7 @@ os__getvolumepathname(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_getvolumepathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getvolumepathname", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1992,7 +1992,7 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_splitroot", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_path_splitroot", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -2024,21 +2024,28 @@ PyDoc_STRVAR(os__path_exists__doc__, {"_path_exists", (PyCFunction)os__path_exists, METH_O, os__path_exists__doc__}, static int -os__path_exists_impl(PyObject *module, PyObject *path); +os__path_exists_impl(PyObject *module, path_t *path); static PyObject * -os__path_exists(PyObject *module, PyObject *path) +os__path_exists(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; + path_t path = PATH_T_INITIALIZE_P("_path_exists", "path", 0, 0, 1, 1); int _return_value; - _return_value = os__path_exists_impl(module, path); + if (!path_converter(arg, &path)) { + goto exit; + } + _return_value = os__path_exists_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2056,21 +2063,28 @@ PyDoc_STRVAR(os__path_lexists__doc__, {"_path_lexists", (PyCFunction)os__path_lexists, METH_O, os__path_lexists__doc__}, static int -os__path_lexists_impl(PyObject *module, PyObject *path); +os__path_lexists_impl(PyObject *module, path_t *path); static PyObject * -os__path_lexists(PyObject *module, PyObject *path) +os__path_lexists(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; + path_t path = PATH_T_INITIALIZE_P("_path_lexists", "path", 0, 0, 1, 1); int _return_value; - _return_value = os__path_lexists_impl(module, path); + if (!path_converter(arg, &path)) { + goto exit; + } + _return_value = os__path_lexists_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2088,7 +2102,7 @@ PyDoc_STRVAR(os__path_isdir__doc__, {"_path_isdir", _PyCFunction_CAST(os__path_isdir), METH_FASTCALL|METH_KEYWORDS, os__path_isdir__doc__}, static int -os__path_isdir_impl(PyObject *module, PyObject *path); +os__path_isdir_impl(PyObject *module, path_t *path); static PyObject * os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2120,21 +2134,26 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isdir", "path", 0, 0, 1, 1); int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - _return_value = os__path_isdir_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isdir_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2152,7 +2171,7 @@ PyDoc_STRVAR(os__path_isfile__doc__, {"_path_isfile", _PyCFunction_CAST(os__path_isfile), METH_FASTCALL|METH_KEYWORDS, os__path_isfile__doc__}, static int -os__path_isfile_impl(PyObject *module, PyObject *path); +os__path_isfile_impl(PyObject *module, path_t *path); static PyObject * os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2184,21 +2203,26 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isfile", "path", 0, 0, 1, 1); int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - _return_value = os__path_isfile_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isfile_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2216,7 +2240,7 @@ PyDoc_STRVAR(os__path_islink__doc__, {"_path_islink", _PyCFunction_CAST(os__path_islink), METH_FASTCALL|METH_KEYWORDS, os__path_islink__doc__}, static int -os__path_islink_impl(PyObject *module, PyObject *path); +os__path_islink_impl(PyObject *module, path_t *path); static PyObject * os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2248,21 +2272,26 @@ os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_islink", "path", 0, 0, 1, 1); int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - _return_value = os__path_islink_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_islink_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2280,7 +2309,7 @@ PyDoc_STRVAR(os__path_isjunction__doc__, {"_path_isjunction", _PyCFunction_CAST(os__path_isjunction), METH_FASTCALL|METH_KEYWORDS, os__path_isjunction__doc__}, static int -os__path_isjunction_impl(PyObject *module, PyObject *path); +os__path_isjunction_impl(PyObject *module, path_t *path); static PyObject * os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2312,21 +2341,26 @@ os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isjunction", "path", 0, 0, 1, 1); int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - _return_value = os__path_isjunction_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isjunction_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2335,13 +2369,16 @@ os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyDoc_STRVAR(os__path_splitroot_ex__doc__, "_path_splitroot_ex($module, /, path)\n" "--\n" -"\n"); +"\n" +"Split a pathname into drive, root and tail.\n" +"\n" +"The tail contains anything after the root."); #define OS__PATH_SPLITROOT_EX_METHODDEF \ {"_path_splitroot_ex", _PyCFunction_CAST(os__path_splitroot_ex), METH_FASTCALL|METH_KEYWORDS, os__path_splitroot_ex__doc__}, static PyObject * -os__path_splitroot_ex_impl(PyObject *module, PyObject *path); +os__path_splitroot_ex_impl(PyObject *module, path_t *path); static PyObject * os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2373,20 +2410,21 @@ os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_splitroot_ex", "path", 0, 1, 1, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("_path_splitroot_ex", "argument 'path'", "str", args[0]); + if (!path_converter(args[0], &path)) { goto exit; } - path = args[0]; - return_value = os__path_splitroot_ex_impl(module, path); + return_value = os__path_splitroot_ex_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2394,13 +2432,13 @@ PyDoc_STRVAR(os__path_normpath__doc__, "_path_normpath($module, /, path)\n" "--\n" "\n" -"Basic path normalization."); +"Normalize path, eliminating double slashes, etc."); #define OS__PATH_NORMPATH_METHODDEF \ {"_path_normpath", _PyCFunction_CAST(os__path_normpath), METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__}, static PyObject * -os__path_normpath_impl(PyObject *module, PyObject *path); +os__path_normpath_impl(PyObject *module, path_t *path); static PyObject * os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2432,16 +2470,21 @@ os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_normpath", "path", 0, 1, 1, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_normpath_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + return_value = os__path_normpath_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2496,7 +2539,7 @@ os_mkdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mkdir", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mkdir", "path", 0, 0, 0, 0); int mode = 511; int dir_fd = DEFAULT_DIR_FD; @@ -2757,8 +2800,8 @@ os_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("rename", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("rename", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("rename", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("rename", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; @@ -2848,8 +2891,8 @@ os_replace(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("replace", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("replace", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("replace", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("replace", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; @@ -2937,7 +2980,7 @@ os_rmdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("rmdir", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("rmdir", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3186,7 +3229,7 @@ os_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("unlink", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("unlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3260,7 +3303,7 @@ os_remove(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("remove", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("remove", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3378,7 +3421,7 @@ os_utime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("utime", "path", 0, PATH_UTIME_HAVE_FD); + path_t path = PATH_T_INITIALIZE_P("utime", "path", 0, 0, 0, PATH_UTIME_HAVE_FD); PyObject *times = Py_None; PyObject *ns = NULL; int dir_fd = DEFAULT_DIR_FD; @@ -3513,7 +3556,7 @@ static PyObject * os_execv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("execv", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("execv", "path", 0, 0, 0, 0); PyObject *argv; if (!_PyArg_CheckPositional("execv", nargs, 2, 2)) { @@ -3585,7 +3628,7 @@ os_execve(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[3]; - path_t path = PATH_T_INITIALIZE("execve", "path", 0, PATH_HAVE_FEXECVE); + path_t path = PATH_T_INITIALIZE_P("execve", "path", 0, 0, 0, PATH_HAVE_FEXECVE); PyObject *argv; PyObject *env; @@ -3681,7 +3724,7 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[10]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("posix_spawn", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("posix_spawn", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; PyObject *file_actions = NULL; @@ -3831,7 +3874,7 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #undef KWTUPLE PyObject *argsbuf[10]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("posix_spawnp", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("posix_spawnp", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; PyObject *file_actions = NULL; @@ -3935,7 +3978,7 @@ os_spawnv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int mode; - path_t path = PATH_T_INITIALIZE("spawnv", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("spawnv", "path", 0, 0, 0, 0); PyObject *argv; if (!_PyArg_CheckPositional("spawnv", nargs, 3, 3)) { @@ -3989,7 +4032,7 @@ os_spawnve(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int mode; - path_t path = PATH_T_INITIALIZE("spawnve", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("spawnve", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; @@ -6165,7 +6208,7 @@ os_readlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("readlink", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("readlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -6249,8 +6292,8 @@ os_symlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("symlink", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("symlink", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("symlink", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("symlink", "dst", 0, 0, 0, 0); int target_is_directory = 0; int dir_fd = DEFAULT_DIR_FD; @@ -6892,7 +6935,7 @@ os_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("open", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("open", "path", 0, 0, 0, 0); int flags; int mode = 511; int dir_fd = DEFAULT_DIR_FD; @@ -8480,7 +8523,7 @@ os_mkfifo(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mkfifo", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mkfifo", "path", 0, 0, 0, 0); int mode = 438; int dir_fd = DEFAULT_DIR_FD; @@ -8580,7 +8623,7 @@ os_mknod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mknod", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mknod", "path", 0, 0, 0, 0); int mode = 384; dev_t device = 0; int dir_fd = DEFAULT_DIR_FD; @@ -8834,7 +8877,7 @@ os_truncate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("truncate", "path", 0, PATH_HAVE_FTRUNCATE); + path_t path = PATH_T_INITIALIZE_P("truncate", "path", 0, 0, 0, PATH_HAVE_FTRUNCATE); Py_off_t length; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -9733,7 +9776,7 @@ os_statvfs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("statvfs", "path", 0, PATH_HAVE_FSTATVFS); + path_t path = PATH_T_INITIALIZE_P("statvfs", "path", 0, 0, 0, PATH_HAVE_FSTATVFS); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -9797,7 +9840,7 @@ os__getdiskusage(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_getdiskusage", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getdiskusage", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -9911,7 +9954,7 @@ os_pathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("pathconf", "path", 0, PATH_HAVE_FPATHCONF); + path_t path = PATH_T_INITIALIZE_P("pathconf", "path", 0, 0, 0, PATH_HAVE_FPATHCONF); int name; long _return_value; @@ -10101,10 +10144,10 @@ os_startfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t filepath = PATH_T_INITIALIZE("startfile", "filepath", 0, 0); + path_t filepath = PATH_T_INITIALIZE_P("startfile", "filepath", 0, 0, 0, 0); const wchar_t *operation = NULL; const wchar_t *arguments = NULL; - path_t cwd = PATH_T_INITIALIZE("startfile", "cwd", 1, 0); + path_t cwd = PATH_T_INITIALIZE_P("startfile", "cwd", 1, 0, 0, 0); int show_cmd = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 5, 0, argsbuf); @@ -10439,8 +10482,8 @@ os_getxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("getxattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("getxattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("getxattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("getxattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -10526,8 +10569,8 @@ os_setxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("setxattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("setxattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("setxattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("setxattr", "attribute", 0, 0, 0, 0); Py_buffer value = {NULL, NULL}; int flags = 0; int follow_symlinks = 1; @@ -10634,8 +10677,8 @@ os_removexattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("removexattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("removexattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("removexattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("removexattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -10720,7 +10763,7 @@ os_listxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("listxattr", "path", 1, 1); + path_t path = PATH_T_INITIALIZE_P("listxattr", "path", 1, 0, 0, 1); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); @@ -11697,7 +11740,7 @@ os_scandir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("scandir", "path", 1, PATH_HAVE_FDOPENDIR); + path_t path = PATH_T_INITIALIZE_P("scandir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { @@ -11909,7 +11952,7 @@ os__add_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_add_dll_directory", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_add_dll_directory", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -12752,4 +12795,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=af5074c4ce4b19f1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=300bd1c54dc43765 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 07fec35cb32d90..bb35cfd9cdb138 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1092,16 +1092,15 @@ get_posix_state(PyObject *module) * * path_converter accepts (Unicode) strings and their * subclasses, and bytes and their subclasses. What - * it does with the argument depends on the platform: + * it does with the argument depends on path.make_wide: * - * * On Windows, if we get a (Unicode) string we - * extract the wchar_t * and return it; if we get - * bytes we decode to wchar_t * and return that. + * * If path.make_wide is nonzero, if we get a (Unicode) + * string we extract the wchar_t * and return it; if we + * get bytes we decode to wchar_t * and return that. * - * * On all other platforms, strings are encoded - * to bytes using PyUnicode_FSConverter, then we - * extract the char * from the bytes object and - * return that. + * * If path.make_wide is zero, if we get bytes we extract + * the char_t * and return it; if we get a (Unicode) + * string we encode to char_t * and return that. * * path_converter also optionally accepts signed * integers (representing open file descriptors) instead @@ -1110,6 +1109,15 @@ get_posix_state(PyObject *module) * Input fields: * path.nullable * If nonzero, the path is permitted to be None. + * path.nonstrict + * If nonzero, the path is permitted to contain + * embedded null characters and have any length. + * path.make_wide + * If nonzero, the converter always uses wide, decoding if necessary, else + * it always uses narrow, encoding if necessary. The default value is + * nonzero on Windows, else zero. + * path.suppress_value_error + * If nonzero, raising ValueError is suppressed. * path.allow_fd * If nonzero, the path is permitted to be a file handle * (a signed int) instead of a string. @@ -1125,12 +1133,10 @@ get_posix_state(PyObject *module) * Output fields: * path.wide * Points to the path if it was expressed as Unicode - * and was not encoded. (Only used on Windows.) + * or if it was bytes and decoded to Unicode. * path.narrow * Points to the path if it was expressed as bytes, - * or it was Unicode and was encoded to bytes. (On Windows, - * is a non-zero integer if the path was expressed as bytes. - * The type is deliberately incompatible to prevent misuse.) + * or if it was Unicode and encoded to bytes. * path.fd * Contains a file descriptor if path.accept_fd was true * and the caller provided a signed integer instead of any @@ -1140,6 +1146,9 @@ get_posix_state(PyObject *module) * unspecified, path_converter will never get called. * So if you set allow_fd, you *MUST* initialize path.fd = -1 * yourself! + * path.value_error + * If nonzero, then suppress_value_error was specified and a ValueError + * occurred. * path.length * The length of the path in characters, if specified as * a string. @@ -1172,28 +1181,38 @@ get_posix_state(PyObject *module) * path_cleanup(). However it is safe to do so.) */ typedef struct { + // Input fields const char *function_name; const char *argument_name; int nullable; + int nonstrict; + int make_wide; + int suppress_value_error; int allow_fd; + // Output fields const wchar_t *wide; -#ifdef MS_WINDOWS - BOOL narrow; -#else const char *narrow; -#endif int fd; + int value_error; Py_ssize_t length; PyObject *object; PyObject *cleanup; } path_t; +#define PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, \ + make_wide, suppress_value_error, allow_fd) \ + {function_name, argument_name, nullable, nonstrict, make_wide, \ + suppress_value_error, allow_fd, NULL, NULL, -1, 0, 0, NULL, NULL} #ifdef MS_WINDOWS -#define PATH_T_INITIALIZE(function_name, argument_name, nullable, allow_fd) \ - {function_name, argument_name, nullable, allow_fd, NULL, FALSE, -1, 0, NULL, NULL} +#define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ + nonstrict, suppress_value_error, allow_fd) \ + PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, 1, \ + suppress_value_error, allow_fd) #else -#define PATH_T_INITIALIZE(function_name, argument_name, nullable, allow_fd) \ - {function_name, argument_name, nullable, allow_fd, NULL, NULL, -1, 0, NULL, NULL} +#define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ + nonstrict, suppress_value_error, allow_fd) \ + PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, 0, \ + suppress_value_error, allow_fd) #endif static void @@ -1214,10 +1233,8 @@ path_converter(PyObject *o, void *p) Py_ssize_t length = 0; int is_index, is_bytes, is_unicode; const char *narrow; -#ifdef MS_WINDOWS PyObject *wo = NULL; wchar_t *wide = NULL; -#endif #define FORMAT_EXCEPTION(exc, fmt) \ PyErr_Format(exc, "%s%s" fmt, \ @@ -1238,11 +1255,7 @@ path_converter(PyObject *o, void *p) if ((o == Py_None) && path->nullable) { path->wide = NULL; -#ifdef MS_WINDOWS - path->narrow = FALSE; -#else path->narrow = NULL; -#endif path->fd = -1; goto success_exit; } @@ -1286,30 +1299,33 @@ path_converter(PyObject *o, void *p) } if (is_unicode) { + if (path->make_wide) { + wide = PyUnicode_AsWideCharString(o, &length); + if (!wide) { + goto error_exit; + } #ifdef MS_WINDOWS - wide = PyUnicode_AsWideCharString(o, &length); - if (!wide) { - goto error_exit; - } - if (length > 32767) { - FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - goto error_exit; - } - if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); - goto error_exit; - } + if (!path->nonstrict && length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + goto error_exit; + } +#endif + if (!path->nonstrict && wcslen(wide) != (size_t)length) { + FORMAT_EXCEPTION(PyExc_ValueError, + "embedded null character in %s"); + goto error_exit; + } - path->wide = wide; - path->narrow = FALSE; - path->fd = -1; - wide = NULL; - goto success_exit; -#else - if (!PyUnicode_FSConverter(o, &bytes)) { + path->wide = wide; + path->narrow = NULL; + path->fd = -1; + wide = NULL; + goto success_exit; + } + bytes = PyUnicode_EncodeFSDefault(o); + if (!bytes) { goto error_exit; } -#endif } else if (is_bytes) { bytes = Py_NewRef(o); @@ -1319,11 +1335,7 @@ path_converter(PyObject *o, void *p) goto error_exit; } path->wide = NULL; -#ifdef MS_WINDOWS - path->narrow = FALSE; -#else path->narrow = NULL; -#endif goto success_exit; } else { @@ -1343,52 +1355,54 @@ path_converter(PyObject *o, void *p) length = PyBytes_GET_SIZE(bytes); narrow = PyBytes_AS_STRING(bytes); - if ((size_t)length != strlen(narrow)) { + if (!path->nonstrict && strlen(narrow) != (size_t)length) { FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); goto error_exit; } -#ifdef MS_WINDOWS - wo = PyUnicode_DecodeFSDefaultAndSize( - narrow, - length - ); - if (!wo) { - goto error_exit; - } + if (path->make_wide) { + wo = PyUnicode_DecodeFSDefaultAndSize(narrow, length); + if (!wo) { + goto error_exit; + } - wide = PyUnicode_AsWideCharString(wo, &length); - Py_DECREF(wo); - if (!wide) { - goto error_exit; - } - if (length > 32767) { - FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - goto error_exit; - } - if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); - goto error_exit; - } - path->wide = wide; - path->narrow = TRUE; - Py_DECREF(bytes); - wide = NULL; -#else - path->wide = NULL; - path->narrow = narrow; - if (bytes == o) { - /* Still a reference owned by path->object, don't have to - worry about path->narrow is used after free. */ + wide = PyUnicode_AsWideCharString(wo, &length); + Py_DECREF(wo); + if (!wide) { + goto error_exit; + } +#ifdef MS_WINDOWS + if (!path->nonstrict && length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + goto error_exit; + } +#endif + if (!path->nonstrict && wcslen(wide) != (size_t)length) { + FORMAT_EXCEPTION(PyExc_ValueError, + "embedded null character in %s"); + goto error_exit; + } + path->wide = wide; + path->narrow = NULL; Py_DECREF(bytes); + wide = NULL; } else { - path->cleanup = bytes; + path->wide = NULL; + path->narrow = narrow; + if (bytes == o) { + /* Still a reference owned by path->object, don't have to + worry about path->narrow is used after free. */ + Py_DECREF(bytes); + } + else { + path->cleanup = bytes; + } } -#endif path->fd = -1; success_exit: + path->value_error = 0; path->length = length; path->object = o; return Py_CLEANUP_SUPPORTED; @@ -1396,10 +1410,20 @@ path_converter(PyObject *o, void *p) error_exit: Py_XDECREF(o); Py_XDECREF(bytes); -#ifdef MS_WINDOWS PyMem_Free(wide); -#endif - return 0; + if (!path->suppress_value_error || + !PyErr_ExceptionMatches(PyExc_ValueError)) + { + return 0; + } + PyErr_Clear(); + path->wide = NULL; + path->narrow = NULL; + path->fd = -1; + path->value_error = 1; + path->length = 0; + path->object = NULL; + return Py_CLEANUP_SUPPORTED; } static void @@ -1449,11 +1473,7 @@ follow_symlinks_specified(const char *function_name, int follow_symlinks) static int path_and_dir_fd_invalid(const char *function_name, path_t *path, int dir_fd) { - if (!path->wide && (dir_fd != DEFAULT_DIR_FD) -#ifndef MS_WINDOWS - && !path->narrow -#endif - ) { + if (!path->wide && (dir_fd != DEFAULT_DIR_FD) && !path->narrow) { PyErr_Format(PyExc_ValueError, "%s: can't specify dir_fd without matching path", function_name); @@ -2913,7 +2933,9 @@ class path_t_converter(CConverter): converter = 'path_converter' - def converter_init(self, *, allow_fd=False, nullable=False): + def converter_init(self, *, allow_fd=False, make_wide=None, + nonstrict=False, nullable=False, + suppress_value_error=False): # right now path_t doesn't support default values. # to support a default value, you'll need to override initialize(). if self.default not in (unspecified, None): @@ -2923,6 +2945,9 @@ class path_t_converter(CConverter): raise RuntimeError("Can't specify a c_default to the path_t converter!") self.nullable = nullable + self.nonstrict = nonstrict + self.make_wide = make_wide + self.suppress_value_error = suppress_value_error self.allow_fd = allow_fd def pre_render(self): @@ -2932,11 +2957,24 @@ class path_t_converter(CConverter): return str(int(bool(value))) # add self.py_name here when merging with posixmodule conversion - self.c_default = 'PATH_T_INITIALIZE("{}", "{}", {}, {})'.format( - self.function.name, - self.name, - strify(self.nullable), - strify(self.allow_fd), + if self.make_wide is None: + self.c_default = 'PATH_T_INITIALIZE_P("{}", "{}", {}, {}, {}, {})'.format( + self.function.name, + self.name, + strify(self.nullable), + strify(self.nonstrict), + strify(self.suppress_value_error), + strify(self.allow_fd), + ) + else: + self.c_default = 'PATH_T_INITIALIZE("{}", "{}", {}, {}, {}, {}, {})'.format( + self.function.name, + self.name, + strify(self.nullable), + strify(self.nonstrict), + strify(self.make_wide), + strify(self.suppress_value_error), + strify(self.allow_fd), ) def cleanup(self): @@ -3016,7 +3054,7 @@ class sysconf_confname_converter(path_confname_converter): converter="conv_sysconf_confname" [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=3338733161aa7879]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=577cb476e5d64960]*/ /*[clinic input] @@ -4285,7 +4323,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) { PyObject *v; HANDLE hFindFile = INVALID_HANDLE_VALUE; - BOOL result; + BOOL result, return_bytes; wchar_t namebuf[MAX_PATH+4]; /* Overallocate for "\*.*" */ /* only claim to have space for MAX_PATH */ Py_ssize_t len = Py_ARRAY_LENGTH(namebuf)-4; @@ -4297,9 +4335,11 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) if (!path->wide) { /* Default arg: "." */ po_wchars = L"."; len = 1; + return_bytes = 0; } else { po_wchars = path->wide; len = wcslen(path->wide); + return_bytes = PyBytes_Check(path->object); } /* The +5 is so we can append "\\*.*\0" */ wnamebuf = PyMem_New(wchar_t, len + 5); @@ -4334,7 +4374,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) wcscmp(wFileData.cFileName, L"..") != 0) { v = PyUnicode_FromWideChar(wFileData.cFileName, wcslen(wFileData.cFileName)); - if (path->narrow && v) { + if (return_bytes && v) { Py_SETREF(v, PyUnicode_EncodeFSDefault(v)); } if (v == NULL) { @@ -4877,7 +4917,7 @@ os__getfullpathname_impl(PyObject *module, path_t *path) if (str == NULL) { return NULL; } - if (path->narrow) { + if (PyBytes_Check(path->object)) { Py_SETREF(str, PyUnicode_EncodeFSDefault(str)); } return str; @@ -4950,7 +4990,7 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) } result = PyUnicode_FromWideChar(target_path, result_length); - if (result && path->narrow) { + if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } @@ -5033,7 +5073,7 @@ os__getvolumepathname_impl(PyObject *module, path_t *path) goto exit; } result = PyUnicode_FromWideChar(mountpath, wcslen(mountpath)); - if (path->narrow) + if (PyBytes_Check(path->object)) Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); exit: @@ -5267,64 +5307,52 @@ _testFileExistsByName(LPCWSTR path, BOOL followLinks) } -static int -_testFileExists(path_t *_path, PyObject *path, BOOL followLinks) +static BOOL +_testFileExists(path_t *path, BOOL followLinks) { BOOL result = FALSE; - if (!path_converter(path, _path)) { - path_cleanup(_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - return FALSE; - } - return -1; + if (path->value_error) { + return FALSE; } Py_BEGIN_ALLOW_THREADS - if (_path->fd != -1) { - HANDLE hfile = _Py_get_osfhandle_noraise(_path->fd); + if (path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(path->fd); if (hfile != INVALID_HANDLE_VALUE) { if (GetFileType(hfile) != FILE_TYPE_UNKNOWN || !GetLastError()) { result = TRUE; } } } - else if (_path->wide) { - result = _testFileExistsByName(_path->wide, followLinks); + else if (path->wide) { + result = _testFileExistsByName(path->wide, followLinks); } Py_END_ALLOW_THREADS - path_cleanup(_path); return result; } -static int -_testFileType(path_t *_path, PyObject *path, int testedType) +static BOOL +_testFileType(path_t *path, int testedType) { BOOL result = FALSE; - if (!path_converter(path, _path)) { - path_cleanup(_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - return FALSE; - } - return -1; + if (path->value_error) { + return FALSE; } Py_BEGIN_ALLOW_THREADS - if (_path->fd != -1) { - HANDLE hfile = _Py_get_osfhandle_noraise(_path->fd); + if (path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(path->fd); if (hfile != INVALID_HANDLE_VALUE) { result = _testFileTypeByHandle(hfile, testedType, TRUE); } } - else if (_path->wide) { - result = _testFileTypeByName(_path->wide, testedType); + else if (path->wide) { + result = _testFileTypeByName(path->wide, testedType); } Py_END_ALLOW_THREADS - path_cleanup(_path); return result; } @@ -5332,7 +5360,7 @@ _testFileType(path_t *_path, PyObject *path, int testedType) /*[clinic input] os._path_exists -> bool - path: object + path: path_t(allow_fd=True, suppress_value_error=True) / Test whether a path exists. Returns False for broken symbolic links. @@ -5340,18 +5368,17 @@ Test whether a path exists. Returns False for broken symbolic links. [clinic start generated code]*/ static int -os__path_exists_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=8f784b3abf9f8588 input=2777da15bc4ba5a3]*/ +os__path_exists_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=8da13acf666e16ba input=29198507a6082a57]*/ { - path_t _path = PATH_T_INITIALIZE("_path_exists", "path", 0, 1); - return _testFileExists(&_path, path, TRUE); + return _testFileExists(path, TRUE); } /*[clinic input] os._path_lexists -> bool - path: object + path: path_t(allow_fd=True, suppress_value_error=True) / Test whether a path exists. Returns True for broken symbolic links. @@ -5359,83 +5386,78 @@ Test whether a path exists. Returns True for broken symbolic links. [clinic start generated code]*/ static int -os__path_lexists_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=fec4a91cf4ffccf1 input=8843d4d6d4e7c779]*/ +os__path_lexists_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=e7240ed5fc45bff3 input=03d9fed8bc6ce96f]*/ { - path_t _path = PATH_T_INITIALIZE("_path_lexists", "path", 0, 1); - return _testFileExists(&_path, path, FALSE); + return _testFileExists(path, FALSE); } /*[clinic input] os._path_isdir -> bool - s as path: object + s as path: path_t(allow_fd=True, suppress_value_error=True) Return true if the pathname refers to an existing directory. [clinic start generated code]*/ static int -os__path_isdir_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=0504fd403f369701 input=2cb54dd97eb970f7]*/ +os__path_isdir_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=d5786196f9e2fa7a input=132a3b5301aecf79]*/ { - path_t _path = PATH_T_INITIALIZE("_path_isdir", "s", 0, 1); - return _testFileType(&_path, path, PY_IFDIR); + return _testFileType(path, PY_IFDIR); } /*[clinic input] os._path_isfile -> bool - path: object + path: path_t(allow_fd=True, suppress_value_error=True) Test whether a path is a regular file [clinic start generated code]*/ static int -os__path_isfile_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b40d620efe5a896f input=54b428a310debaea]*/ +os__path_isfile_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=5c3073bc212b9863 input=4ac1fd350b30a39e]*/ { - path_t _path = PATH_T_INITIALIZE("_path_isfile", "path", 0, 1); - return _testFileType(&_path, path, PY_IFREG); + return _testFileType(path, PY_IFREG); } /*[clinic input] os._path_islink -> bool - path: object + path: path_t(allow_fd=True, suppress_value_error=True) Test whether a path is a symbolic link [clinic start generated code]*/ static int -os__path_islink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=9d0cf8e4c640dfe6 input=b71fed60b9b2cd73]*/ +os__path_islink_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=30da7bda8296adcc input=7510ce05b547debb]*/ { - path_t _path = PATH_T_INITIALIZE("_path_islink", "path", 0, 1); - return _testFileType(&_path, path, PY_IFLNK); + return _testFileType(path, PY_IFLNK); } /*[clinic input] os._path_isjunction -> bool - path: object + path: path_t(allow_fd=True, suppress_value_error=True) Test whether a path is a junction [clinic start generated code]*/ static int -os__path_isjunction_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=f1d51682a077654d input=103ccedcdb714f11]*/ +os__path_isjunction_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=e1d17a9dd18a9945 input=7dcb8bc4e972fcaf]*/ { - path_t _path = PATH_T_INITIALIZE("_path_isjunction", "path", 0, 1); - return _testFileType(&_path, path, PY_IFMNT); + return _testFileType(path, PY_IFMNT); } #undef PY_IFREG @@ -5451,23 +5473,22 @@ os__path_isjunction_impl(PyObject *module, PyObject *path) /*[clinic input] os._path_splitroot_ex - path: unicode + path: path_t(make_wide=True, nonstrict=True) +Split a pathname into drive, root and tail. + +The tail contains anything after the root. [clinic start generated code]*/ static PyObject * -os__path_splitroot_ex_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=de97403d3dfebc40 input=f1470e12d899f9ac]*/ +os__path_splitroot_ex_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=4b0072b6cdf4b611 input=6eb76e9173412c92]*/ { - Py_ssize_t len, drvsize, rootsize; + Py_ssize_t drvsize, rootsize; PyObject *drv = NULL, *root = NULL, *tail = NULL, *result = NULL; - wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); - if (!buffer) { - goto exit; - } - - _Py_skiproot(buffer, len, &drvsize, &rootsize); + const wchar_t *buffer = path->wide; + _Py_skiproot(buffer, path->length, &drvsize, &rootsize); drv = PyUnicode_FromWideChar(buffer, drvsize); if (drv == NULL) { goto exit; @@ -5477,13 +5498,26 @@ os__path_splitroot_ex_impl(PyObject *module, PyObject *path) goto exit; } tail = PyUnicode_FromWideChar(&buffer[drvsize + rootsize], - len - drvsize - rootsize); + path->length - drvsize - rootsize); if (tail == NULL) { goto exit; } + if (PyBytes_Check(path->object)) { + Py_SETREF(drv, PyUnicode_EncodeFSDefault(drv)); + if (drv == NULL) { + goto exit; + } + Py_SETREF(root, PyUnicode_EncodeFSDefault(root)); + if (root == NULL) { + goto exit; + } + Py_SETREF(tail, PyUnicode_EncodeFSDefault(tail)); + if (tail == NULL) { + goto exit; + } + } result = PyTuple_Pack(3, drv, root, tail); exit: - PyMem_Free(buffer); Py_XDECREF(drv); Py_XDECREF(root); Py_XDECREF(tail); @@ -5494,29 +5528,28 @@ os__path_splitroot_ex_impl(PyObject *module, PyObject *path) /*[clinic input] os._path_normpath - path: object + path: path_t(make_wide=True, nonstrict=True) -Basic path normalization. +Normalize path, eliminating double slashes, etc. [clinic start generated code]*/ static PyObject * -os__path_normpath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/ +os__path_normpath_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=d353e7ed9410c044 input=3d4ac23b06332dcb]*/ { - if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", - Py_TYPE(path)->tp_name); - return NULL; + PyObject *result; + Py_ssize_t norm_len; + wchar_t *norm_path = _Py_normpath_and_size((wchar_t *)path->wide, + path->length, &norm_len); + if (!norm_len) { + result = PyUnicode_FromOrdinal('.'); } - Py_ssize_t len; - wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); - if (!buffer) { - return NULL; + else { + result = PyUnicode_FromWideChar(norm_path, norm_len); + } + if (PyBytes_Check(path->object)) { + Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } - Py_ssize_t norm_len; - wchar_t *norm_path = _Py_normpath_and_size(buffer, len, &norm_len); - PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); - PyMem_Free(buffer); return result; } @@ -10243,7 +10276,7 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd) name[1] = L'\\'; } result = PyUnicode_FromWideChar(name, nameLen); - if (result && path->narrow) { + if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } } @@ -15864,7 +15897,8 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) entry->name = PyUnicode_FromWideChar(dataW->cFileName, -1); if (!entry->name) goto error; - if (path->narrow) { + int return_bytes = path->wide && PyBytes_Check(path->object); + if (return_bytes) { Py_SETREF(entry->name, PyUnicode_EncodeFSDefault(entry->name)); if (!entry->name) goto error; @@ -15878,7 +15912,7 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) PyMem_Free(joined_path); if (!entry->path) goto error; - if (path->narrow) { + if (return_bytes) { Py_SETREF(entry->path, PyUnicode_EncodeFSDefault(entry->path)); if (!entry->path) goto error; From 045e195c76f33c77c339284b13f81102e4b9abe2 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Fri, 24 May 2024 22:30:32 +0300 Subject: [PATCH 201/903] Regen ``Doc/requirements-oldest-sphinx.txt`` (#119520) --- Doc/requirements-oldest-sphinx.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt index fe45c91501a56a..3ae65bc944da26 100644 --- a/Doc/requirements-oldest-sphinx.txt +++ b/Doc/requirements-oldest-sphinx.txt @@ -23,7 +23,7 @@ Jinja2==3.1.4 MarkupSafe==2.1.5 packaging==24.0 Pygments==2.18.0 -requests==2.31.0 +requests==2.32.2 snowballstemmer==2.2.0 Sphinx==6.2.1 sphinxcontrib-applehelp==1.0.8 From 81d63362302187e5cb838c9a7cd857181142e530 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 24 May 2024 20:35:13 +0100 Subject: [PATCH 202/903] GH-119054: Add "Querying file type and status" section to pathlib docs (#119055) Add a dedicated subsection for `Path.stat()`-related methods, specifically `stat()`, `lstat()`, `exists()`, `is_*()`, and `samefile()`. --- Doc/library/pathlib.rst | 338 ++++++++++++++++++++-------------------- 1 file changed, 171 insertions(+), 167 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 27ed0a32e801cc..71e2e5452d1754 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -808,8 +808,8 @@ bugs or failures in your application):: UnsupportedOperation: cannot instantiate 'WindowsPath' on your system -File URIs -^^^^^^^^^ +Parsing and generating URIs +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Concrete path objects can be created from, and represented as, 'file' URIs conforming to :rfc:`8089`. @@ -869,12 +869,8 @@ conforming to :rfc:`8089`. it strictly impure. -Methods -^^^^^^^ - -Concrete paths provide the following methods in addition to pure paths -methods. Some of these methods can raise an :exc:`OSError` if a system -call fails (for example because the path doesn't exist). +Querying file type and status +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. versionchanged:: 3.8 @@ -895,29 +891,6 @@ call fails (for example because the path doesn't exist). status without suppressing exceptions. -.. classmethod:: Path.cwd() - - Return a new path object representing the current directory (as returned - by :func:`os.getcwd`):: - - >>> Path.cwd() - PosixPath('/home/antoine/pathlib') - - -.. classmethod:: Path.home() - - Return a new path object representing the user's home directory (as - returned by :func:`os.path.expanduser` with ``~`` construct). If the home - directory can't be resolved, :exc:`RuntimeError` is raised. - - :: - - >>> Path.home() - PosixPath('/home/antoine') - - .. versionadded:: 3.5 - - .. method:: Path.stat(*, follow_symlinks=True) Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. @@ -937,25 +910,12 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.10 The *follow_symlinks* parameter was added. -.. method:: Path.chmod(mode, *, follow_symlinks=True) - - Change the file mode and permissions, like :func:`os.chmod`. - - This method normally follows symlinks. Some Unix flavours support changing - permissions on the symlink itself; on these platforms you may add the - argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. - :: +.. method:: Path.lstat() - >>> p = Path('setup.py') - >>> p.stat().st_mode - 33277 - >>> p.chmod(0o444) - >>> p.stat().st_mode - 33060 + Like :meth:`Path.stat` but, if the path points to a symbolic link, return + the symbolic link's information rather than its target's. - .. versionchanged:: 3.10 - The *follow_symlinks* parameter was added. .. method:: Path.exists(*, follow_symlinks=True) @@ -980,6 +940,170 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.12 The *follow_symlinks* parameter was added. + +.. method:: Path.is_file(*, follow_symlinks=True) + + Return ``True`` if the path points to a regular file. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a regular file. Use :meth:`Path.stat` to + distinguish between these cases. + + This method normally follows symlinks; to exclude symlinks, add the + argument ``follow_symlinks=False``. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + + +.. method:: Path.is_dir(*, follow_symlinks=True) + + Return ``True`` if the path points to a directory. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a directory. Use :meth:`Path.stat` to distinguish + between these cases. + + This method normally follows symlinks; to exclude symlinks to directories, + add the argument ``follow_symlinks=False``. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + + +.. method:: Path.is_symlink() + + Return ``True`` if the path points to a symbolic link, even if that symlink + is broken. ``False`` will be returned if the path is invalid, inaccessible + or missing, or if it points to something other than a symbolic link. Use + :meth:`Path.stat` to distinguish between these cases. + + +.. method:: Path.is_junction() + + Return ``True`` if the path points to a junction, and ``False`` for any other + type of file. Currently only Windows supports junctions. + + .. versionadded:: 3.12 + + +.. method:: Path.is_mount() + + Return ``True`` if the path is a :dfn:`mount point`: a point in a + file system where a different file system has been mounted. On POSIX, the + function checks whether *path*'s parent, :file:`path/..`, is on a different + device than *path*, or whether :file:`path/..` and *path* point to the same + i-node on the same device --- this should detect mount points for all Unix + and POSIX variants. On Windows, a mount point is considered to be a drive + letter root (e.g. ``c:\``), a UNC share (e.g. ``\\server\share``), or a + mounted filesystem directory. + + .. versionadded:: 3.7 + + .. versionchanged:: 3.12 + Windows support was added. + +.. method:: Path.is_socket() + + Return ``True`` if the path points to a Unix socket. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a Unix socket. Use :meth:`Path.stat` to + distinguish between these cases. + + +.. method:: Path.is_fifo() + + Return ``True`` if the path points to a FIFO. ``False`` will be returned if + the path is invalid, inaccessible or missing, or if it points to something + other than a FIFO. Use :meth:`Path.stat` to distinguish between these + cases. + + +.. method:: Path.is_block_device() + + Return ``True`` if the path points to a block device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a block device. Use :meth:`Path.stat` to + distinguish between these cases. + + +.. method:: Path.is_char_device() + + Return ``True`` if the path points to a character device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a character device. Use :meth:`Path.stat` to + distinguish between these cases. + + +.. method:: Path.samefile(other_path) + + Return whether this path points to the same file as *other_path*, which + can be either a Path object, or a string. The semantics are similar + to :func:`os.path.samefile` and :func:`os.path.samestat`. + + An :exc:`OSError` can be raised if either file cannot be accessed for some + reason. + + :: + + >>> p = Path('spam') + >>> q = Path('eggs') + >>> p.samefile(q) + False + >>> p.samefile('spam') + True + + .. versionadded:: 3.5 + + +Other methods +^^^^^^^^^^^^^ + +Some of these methods can raise an :exc:`OSError` if a system call fails (for +example because the path doesn't exist). + + +.. classmethod:: Path.cwd() + + Return a new path object representing the current directory (as returned + by :func:`os.getcwd`):: + + >>> Path.cwd() + PosixPath('/home/antoine/pathlib') + + +.. classmethod:: Path.home() + + Return a new path object representing the user's home directory (as + returned by :func:`os.path.expanduser` with ``~`` construct). If the home + directory can't be resolved, :exc:`RuntimeError` is raised. + + :: + + >>> Path.home() + PosixPath('/home/antoine') + + .. versionadded:: 3.5 + + +.. method:: Path.chmod(mode, *, follow_symlinks=True) + + Change the file mode and permissions, like :func:`os.chmod`. + + This method normally follows symlinks. Some Unix flavours support changing + permissions on the symlink itself; on these platforms you may add the + argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. + + :: + + >>> p = Path('setup.py') + >>> p.stat().st_mode + 33277 + >>> p.chmod(0o444) + >>> p.stat().st_mode + 33060 + + .. versionchanged:: 3.10 + The *follow_symlinks* parameter was added. + .. method:: Path.expanduser() Return a new path with expanded ``~`` and ``~user`` constructs, @@ -1076,99 +1200,6 @@ call fails (for example because the path doesn't exist). The *follow_symlinks* parameter was added. -.. method:: Path.is_dir(*, follow_symlinks=True) - - Return ``True`` if the path points to a directory. ``False`` will be - returned if the path is invalid, inaccessible or missing, or if it points - to something other than a directory. Use :meth:`Path.stat` to distinguish - between these cases. - - This method normally follows symlinks; to exclude symlinks to directories, - add the argument ``follow_symlinks=False``. - - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. - - -.. method:: Path.is_file(*, follow_symlinks=True) - - Return ``True`` if the path points to a regular file. ``False`` will be - returned if the path is invalid, inaccessible or missing, or if it points - to something other than a regular file. Use :meth:`Path.stat` to - distinguish between these cases. - - This method normally follows symlinks; to exclude symlinks, add the - argument ``follow_symlinks=False``. - - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. - - -.. method:: Path.is_junction() - - Return ``True`` if the path points to a junction, and ``False`` for any other - type of file. Currently only Windows supports junctions. - - .. versionadded:: 3.12 - - -.. method:: Path.is_mount() - - Return ``True`` if the path is a :dfn:`mount point`: a point in a - file system where a different file system has been mounted. On POSIX, the - function checks whether *path*'s parent, :file:`path/..`, is on a different - device than *path*, or whether :file:`path/..` and *path* point to the same - i-node on the same device --- this should detect mount points for all Unix - and POSIX variants. On Windows, a mount point is considered to be a drive - letter root (e.g. ``c:\``), a UNC share (e.g. ``\\server\share``), or a - mounted filesystem directory. - - .. versionadded:: 3.7 - - .. versionchanged:: 3.12 - Windows support was added. - - -.. method:: Path.is_symlink() - - Return ``True`` if the path points to a symbolic link, even if that symlink - is broken. ``False`` will be returned if the path is invalid, inaccessible - or missing, or if it points to something other than a symbolic link. Use - :meth:`Path.stat` to distinguish between these cases. - - -.. method:: Path.is_socket() - - Return ``True`` if the path points to a Unix socket. ``False`` will be - returned if the path is invalid, inaccessible or missing, or if it points - to something other than a Unix socket. Use :meth:`Path.stat` to - distinguish between these cases. - - -.. method:: Path.is_fifo() - - Return ``True`` if the path points to a FIFO. ``False`` will be returned if - the path is invalid, inaccessible or missing, or if it points to something - other than a FIFO. Use :meth:`Path.stat` to distinguish between these - cases. - - -.. method:: Path.is_block_device() - - Return ``True`` if the path points to a block device. ``False`` will be - returned if the path is invalid, inaccessible or missing, or if it points - to something other than a block device. Use :meth:`Path.stat` to - distinguish between these cases. - - -.. method:: Path.is_char_device() - - Return ``True`` if the path points to a character device. ``False`` will be - returned if the path is invalid, inaccessible or missing, or if it points - to something other than a character device. Use :meth:`Path.stat` to - distinguish between these cases. - - .. method:: Path.iterdir() When the path points to a directory, yield path objects of the directory @@ -1291,12 +1322,6 @@ call fails (for example because the path doesn't exist). symbolic link's mode is changed rather than its target's. -.. method:: Path.lstat() - - Like :meth:`Path.stat` but, if the path points to a symbolic link, return - the symbolic link's information rather than its target's. - - .. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) Create a new directory at this given path. If *mode* is given, it is @@ -1486,27 +1511,6 @@ call fails (for example because the path doesn't exist). Remove this directory. The directory must be empty. -.. method:: Path.samefile(other_path) - - Return whether this path points to the same file as *other_path*, which - can be either a Path object, or a string. The semantics are similar - to :func:`os.path.samefile` and :func:`os.path.samestat`. - - An :exc:`OSError` can be raised if either file cannot be accessed for some - reason. - - :: - - >>> p = Path('spam') - >>> q = Path('eggs') - >>> p.samefile(q) - False - >>> p.samefile('spam') - True - - .. versionadded:: 3.5 - - .. method:: Path.symlink_to(target, target_is_directory=False) Make this path a symbolic link pointing to *target*. From 49c3ade4f3ceae2f8fcbe03ebaaad5eddf8de0bf Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 24 May 2024 16:58:24 -0500 Subject: [PATCH 203/903] Misc improvement to the docs for itertools (gh-119529) --- Doc/library/itertools.rst | 110 +++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 6d33748898361d..43432dae1623ce 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -134,7 +134,7 @@ loops that truncate the stream. total = func(total, element) yield total - There are a number of uses for the *func* argument. It can be set to + The *func* argument can be set to :func:`min` for a running minimum, :func:`max` for a running maximum, or :func:`operator.mul` for a running product. Amortization tables can be built by accumulating interest and applying payments: @@ -184,21 +184,14 @@ loops that truncate the stream. >>> unflattened [('roses', 'red'), ('violets', 'blue'), ('sugar', 'sweet')] - >>> for batch in batched('ABCDEFG', 3): - ... print(batch) - ... - ('A', 'B', 'C') - ('D', 'E', 'F') - ('G',) - Roughly equivalent to:: def batched(iterable, n, *, strict=False): # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') - it = iter(iterable) - while batch := tuple(islice(it, n)): + iterable = iter(iterable) + while batch := tuple(islice(iterable, n)): if strict and len(batch) != n: raise ValueError('batched(): incomplete batch') yield batch @@ -237,13 +230,13 @@ loops that truncate the stream. Return *r* length subsequences of elements from the input *iterable*. - The combination tuples are emitted in lexicographic ordering according to + The combination tuples are emitted in lexicographic order according to the order of the input *iterable*. So, if the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated - values in each combination. + value. So, if the input elements are unique, there will be no repeated + values within each combination. Roughly equivalent to:: @@ -286,12 +279,12 @@ loops that truncate the stream. Return *r* length subsequences of elements from the input *iterable* allowing individual elements to be repeated more than once. - The combination tuples are emitted in lexicographic ordering according to + The combination tuples are emitted in lexicographic order according to the order of the input *iterable*. So, if the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, the generated combinations + value. So, if the input elements are unique, the generated combinations will also be unique. Roughly equivalent to:: @@ -332,13 +325,13 @@ loops that truncate the stream. .. function:: compress(data, selectors) Make an iterator that filters elements from *data* returning only those that - have a corresponding element in *selectors* that evaluates to ``True``. - Stops when either the *data* or *selectors* iterables has been exhausted. + have a corresponding element in *selectors* is true. + Stops when either the *data* or *selectors* iterables have been exhausted. Roughly equivalent to:: def compress(data, selectors): # compress('ABCDEF', [1,0,1,0,1,1]) → A C E F - return (d for d, s in zip(data, selectors) if s) + return (datum for datum, selector in zip(data, selectors) if selector) .. versionadded:: 3.1 @@ -392,7 +385,7 @@ loops that truncate the stream. start-up time. Roughly equivalent to:: def dropwhile(predicate, iterable): - # dropwhile(lambda x: x<5, [1,4,6,4,1]) → 6 4 1 + # dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8 iterable = iter(iterable) for x in iterable: if not predicate(x): @@ -408,7 +401,7 @@ loops that truncate the stream. that are false. Roughly equivalent to:: def filterfalse(predicate, iterable): - # filterfalse(lambda x: x%2, range(10)) → 0 2 4 6 8 + # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 if predicate is None: predicate = bool for x in iterable: @@ -444,36 +437,37 @@ loops that truncate the stream. :func:`groupby` is roughly equivalent to:: - class groupby: + def groupby(iterable, key=None): # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D - def __init__(self, iterable, key=None): - if key is None: - key = lambda x: x - self.keyfunc = key - self.it = iter(iterable) - self.tgtkey = self.currkey = self.currvalue = object() - - def __iter__(self): - return self - - def __next__(self): - self.id = object() - while self.currkey == self.tgtkey: - self.currvalue = next(self.it) # Exit on StopIteration - self.currkey = self.keyfunc(self.currvalue) - self.tgtkey = self.currkey - return (self.currkey, self._grouper(self.tgtkey, self.id)) - - def _grouper(self, tgtkey, id): - while self.id is id and self.currkey == tgtkey: - yield self.currvalue - try: - self.currvalue = next(self.it) - except StopIteration: + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: return - self.currkey = self.keyfunc(self.currvalue) + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass .. function:: islice(iterable, stop) @@ -501,13 +495,15 @@ loops that truncate the stream. # islice('ABCDEFG', 2, 4) → C D # islice('ABCDEFG', 2, None) → C D E F G # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) start = 0 if s.start is None else s.start stop = s.stop step = 1 if s.step is None else s.step if start < 0 or (stop is not None and stop < 0) or step <= 0: raise ValueError - indices = count() if stop is None else range(max(stop, start)) + + indices = count() if stop is None else range(max(start, stop)) next_i = start for i, element in zip(indices, iterable): if i == next_i: @@ -549,7 +545,7 @@ loops that truncate the stream. the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated + value. So, if the input elements are unique, there will be no repeated values within a permutation. Roughly equivalent to:: @@ -557,14 +553,17 @@ loops that truncate the stream. def permutations(iterable, r=None): # permutations('ABCD', 2) → AB AC AD BA BC BD CA CB CD DA DB DC # permutations(range(3)) → 012 021 102 120 201 210 + pool = tuple(iterable) n = len(pool) r = n if r is None else r if r > n: return + indices = list(range(n)) cycles = list(range(n, n-r, -1)) yield tuple(pool[i] for i in indices[:r]) + while n: for i in reversed(range(r)): cycles[i] -= 1 @@ -580,7 +579,7 @@ loops that truncate the stream. return The code for :func:`permutations` can be also expressed as a subsequence of - :func:`product`, filtered to exclude entries with repeated elements (those + :func:`product` filtered to exclude entries with repeated elements (those from the same position in the input pool):: def permutations(iterable, r=None): @@ -674,17 +673,16 @@ loops that truncate the stream. predicate is true. Roughly equivalent to:: def takewhile(predicate, iterable): - # takewhile(lambda x: x<5, [1,4,6,4,1]) → 1 4 + # takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4 for x in iterable: - if predicate(x): - yield x - else: + if not predicate(x): break + yield x Note, the element that first fails the predicate condition is consumed from the input iterator and there is no way to access it. This could be an issue if an application wants to further consume the - input iterator after takewhile has been run to exhaustion. To work + input iterator after *takewhile* has been run to exhaustion. To work around this problem, consider using `more-iterools before_and_after() `_ instead. @@ -734,10 +732,12 @@ loops that truncate the stream. def zip_longest(*iterables, fillvalue=None): # zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D- - iterators = [iter(it) for it in iterables] + + iterators = list(map(iter, iterables)) num_active = len(iterators) if not num_active: return + while True: values = [] for i, iterator in enumerate(iterators): From 84be5244de75c92904fb41326c9a69f19051e7ab Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 24 May 2024 17:32:17 -0700 Subject: [PATCH 204/903] gh-119180: Update the magic number (#119397) PR #119321 added a comment about the magic number bump but did not actually apply the new magic number. --- Lib/importlib/_bootstrap_external.py | 2 +- PC/launcher.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index b3abf380a82b11..68469863e7f774 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -489,7 +489,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3571).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3600).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/PC/launcher.c b/PC/launcher.c index 8e60ab9303cb95..47fafbc3bf6bad 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1271,6 +1271,7 @@ static PYC_MAGIC magic_values[] = { { 3450, 3499, L"3.11" }, { 3500, 3549, L"3.12" }, { 3550, 3599, L"3.13" }, + { 3600, 3649, L"3.14" }, { 0 } }; From de19694cfbcaa1c85c3a4b7184a24ff21b1c0919 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Fri, 24 May 2024 22:08:21 -0500 Subject: [PATCH 205/903] gh-119105: Differ.compare is too slow [for degenerate cases] (#119492) ``_fancy_replace()`` is no longer recursive. and a single call does a worst-case linear number of ratio() computations instead of quadratic. This renders toothless a universe of pathological cases. Some inputs may produce different output, but that's rare, and I didn't find a case where the final diff appeared to be of materially worse quality. To the contrary, by refusing to even consider synching on lines "far apart", there was more easy-to-digest locality in the output. --- Lib/difflib.py | 116 +++++++----------- ...-05-24-04-05-37.gh-issue-119105.aDSRFn.rst | 1 + 2 files changed, 46 insertions(+), 71 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst diff --git a/Lib/difflib.py b/Lib/difflib.py index 79b446c2afbdc6..0443963b4fd697 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -908,79 +908,52 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): + abcdefGhijkl ? ^ ^ ^ """ - from operator import ge, gt - # Don't synch up unless the lines have a similarity score of at - # least cutoff; best_ratio tracks the best score seen so far. - # Keep track of all index pairs achieving the best ratio and - # deal with them here. Previously only the smallest pair was - # handled here, and if there are many pairs with the best ratio, - # recursion could grow very deep, and runtime cubic. See: + # Don't synch up unless the lines have a similarity score above + # cutoff. Previously only the smallest pair was handled here, + # and if there are many pairs with the best ratio, recursion + # could grow very deep, and runtime cubic. See: # https://github.com/python/cpython/issues/119105 - best_ratio, cutoff = 0.74, 0.75 + # + # Later, more pathological cases prompted removing recursion + # entirely. + cutoff = 0.74999 cruncher = SequenceMatcher(self.charjunk) - eqi, eqj = None, None # 1st indices of equal lines (if any) - # List of index pairs achieving best_ratio. Strictly increasing - # in both index positions. - max_pairs = [] - maxi = -1 # `i` index of last pair in max_pairs - - # search for the pair that matches best without being identical - # (identical lines must be junk lines, & we don't want to synch - # up on junk -- unless we have to) crqr = cruncher.real_quick_ratio cqr = cruncher.quick_ratio cr = cruncher.ratio + + WINDOW = 10 + best_i = best_j = None + dump_i, dump_j = alo, blo # smallest indices not yet resolved for j in range(blo, bhi): - bj = b[j] - cruncher.set_seq2(bj) - # Find new best, if possible. Else search for the smallest i - # (if any) > maxi that equals the best ratio - search_equal = True - for i in range(alo, ahi): - ai = a[i] - if ai == bj: - if eqi is None: - eqi, eqj = i, j - continue - cruncher.set_seq1(ai) - # computing similarity is expensive, so use the quick - # upper bounds first -- have seen this speed up messy - # compares by a factor of 3. - cmp = ge if search_equal and i > maxi else gt - if (cmp(crqr(), best_ratio) - and cmp(cqr(), best_ratio) - and cmp((ratio := cr()), best_ratio)): - if ratio > best_ratio: - best_ratio = ratio - max_pairs.clear() - else: - assert best_ratio == ratio and search_equal - assert i > maxi - max_pairs.append((i, j)) - maxi = i - search_equal = False - if best_ratio < cutoff: - assert not max_pairs - # no non-identical "pretty close" pair - if eqi is None: - # no identical pair either -- treat it as a straight replace - yield from self._plain_replace(a, alo, ahi, b, blo, bhi) - return - # no close pair, but an identical pair -- synch up on that - max_pairs = [(eqi, eqj)] - else: - # there's a close pair, so forget the identical pair (if any) - assert max_pairs - eqi = None - - last_i, last_j = alo, blo - for this_i, this_j in max_pairs: - # pump out diffs from before the synch point - yield from self._fancy_helper(a, last_i, this_i, - b, last_j, this_j) + cruncher.set_seq2(b[j]) + # Search the corresponding i's within WINDOW for rhe highest + # ratio greater than `cutoff`. + aequiv = alo + (j - blo) + arange = range(max(aequiv - WINDOW, dump_i), + min(aequiv + WINDOW + 1, ahi)) + if not arange: # likely exit if `a` is shorter than `b` + break + best_ratio = cutoff + for i in arange: + cruncher.set_seq1(a[i]) + # Ordering by cheapest to most expensive ratio is very + # valuable, most often getting out early. + if (crqr() > best_ratio + and cqr() > best_ratio + and cr() > best_ratio): + best_i, best_j, best_ratio = i, j, cr() + + if best_i is None: + # found nothing to synch on yet - move to next j + continue + + # pump out straight replace from before this synch pair + yield from self._fancy_helper(a, dump_i, best_i, + b, dump_j, best_j) # do intraline marking on the synch pair - aelt, belt = a[this_i], b[this_j] - if eqi is None: + aelt, belt = a[best_i], b[best_j] + if aelt != belt: # pump out a '-', '?', '+', '?' quad for the synched lines atags = btags = "" cruncher.set_seqs(aelt, belt) @@ -1002,17 +975,18 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): else: # the synch pair is identical yield ' ' + aelt - last_i, last_j = this_i + 1, this_j + 1 + dump_i, dump_j = best_i + 1, best_j + 1 + best_i = best_j = None - # pump out diffs from after the last synch point - yield from self._fancy_helper(a, last_i, ahi, - b, last_j, bhi) + # pump out straight replace from after the last synch pair + yield from self._fancy_helper(a, dump_i, ahi, + b, dump_j, bhi) def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] if alo < ahi: if blo < bhi: - g = self._fancy_replace(a, alo, ahi, b, blo, bhi) + g = self._plain_replace(a, alo, ahi, b, blo, bhi) else: g = self._dump('-', a, alo, ahi) elif blo < bhi: diff --git a/Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst b/Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst new file mode 100644 index 00000000000000..3205061a68ce7f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst @@ -0,0 +1 @@ +``difflib``'s ``DIffer.compare()`` (and so also ``ndiff``) can no longer be provoked into cubic-time behavior, or into unbounded recursion, and should generally be faster in ordinary cases too. Results may change in some cases, although that should be rare. Correctness of diffs is not affected. Some similar lines far apart may be reported as deleting one and adding the other, where before they were displayed on adjacent output lines with markup showing the intraline differences. From 08e65430aafa1047029e6f132a5f748c415bda14 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 25 May 2024 16:21:11 +0300 Subject: [PATCH 206/903] gh-111999: Fix the signature of str.format_map() (#119540) --- Doc/library/stdtypes.rst | 2 +- .../2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst | 1 + Objects/unicodeobject.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c0a3d0b3a2a49e..c8acde8b57dcdb 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1768,7 +1768,7 @@ expression support in the :mod:`re` module). cases. -.. method:: str.format_map(mapping) +.. method:: str.format_map(mapping, /) Similar to ``str.format(**mapping)``, except that ``mapping`` is used directly and not copied to a :class:`dict`. This is useful diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst new file mode 100644 index 00000000000000..4b1ca6ca5b0765 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst @@ -0,0 +1 @@ +Fix the signature of :meth:`str.format_map`. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index afff37467caf32..048f9a814c30af 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13490,7 +13490,7 @@ Return a formatted version of the string, using substitutions from args and kwar The substitutions are identified by braces ('{' and '}')."); PyDoc_STRVAR(format_map__doc__, - "format_map($self, /, mapping)\n\ + "format_map($self, mapping, /)\n\ --\n\ \n\ Return a formatted version of the string, using substitutions from mapping.\n\ From a531fd7fdb45d13825cb0c38d97fd38246cf9634 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 25 May 2024 17:13:17 +0300 Subject: [PATCH 207/903] FAQ: Add reference to Python version numbering scheme (#119225) --- Doc/faq/general.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index ec7c2897594999..eb859c5d5992da 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -122,6 +122,8 @@ available. Consult `the Python Package Index `_ to find packages of interest to you. +.. _faq-version-numbering-scheme: + How does the Python version numbering scheme work? -------------------------------------------------- @@ -183,8 +185,6 @@ information on getting the source code and compiling it. How do I get documentation on Python? ------------------------------------- -.. XXX mention py3k - The standard documentation for the current stable version of Python is available at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. From e3bac04c37f6823cebc74d97feae0e0c25818b31 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 25 May 2024 17:15:54 +0100 Subject: [PATCH 208/903] gh-119548: Add a 'clear' command to the REPL (#119549) --- Lib/_pyrepl/reader.py | 5 +++++ Lib/_pyrepl/simple_interact.py | 7 ++++++- .../2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 81df0c925ee6cb..d2960bbb6121b3 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -238,6 +238,7 @@ class Reader: cxy: tuple[int, int] = field(init=False) lxy: tuple[int, int] = field(init=False) calc_screen: CalcScreen = field(init=False) + scheduled_commands: list[str] = field(default_factory=list) def __post_init__(self) -> None: # Enable the use of `insert` without a `prepare` call - necessary to @@ -557,6 +558,10 @@ def prepare(self) -> None: self.restore() raise + while self.scheduled_commands: + cmd = self.scheduled_commands.pop() + self.do_cmd((cmd, [])) + def last_command_is(self, cls: type) -> bool: if not self.last_command: return False diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 8ab4dab757685e..975533a425be23 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -57,12 +57,17 @@ def _strip_final_indent(text: str) -> str: return text +def _clear_screen(): + reader = _get_reader() + reader.scheduled_commands.append("clear_screen") + + REPL_COMMANDS = { "exit": _sitebuiltins.Quitter('exit', ''), "quit": _sitebuiltins.Quitter('quit' ,''), "copyright": _sitebuiltins._Printer('copyright', sys.copyright), "help": "help", - "clear": "clear_screen", + "clear": _clear_screen, } class InteractiveColoredConsole(code.InteractiveConsole): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst new file mode 100644 index 00000000000000..0318790d46f0a3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst @@ -0,0 +1 @@ +Add a ``clear`` command to the REPL. Patch by Pablo Galindo From 6b6c1a904f6d6237a05057727360fe4b80e98d4c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 25 May 2024 17:21:07 +0100 Subject: [PATCH 209/903] Add codeowners for PYREPL (#119550) --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1f5f7e57dc4859..e955567ec0b0f8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -39,9 +39,11 @@ Python/bytecodes.c @markshannon @gvanrossum Python/optimizer*.c @markshannon @gvanrossum Python/optimizer_analysis.c @Fidget-Spinner Python/optimizer_bytecodes.c @Fidget-Spinner +Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv Lib/test/test_patma.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra Lib/test/test_capi/test_misc.py @markshannon @gvanrossum +Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv Tools/c-analyzer/ @ericsnowcurrently # dbm From 4b7eb321bc43e41371df86fce47bd999ee51a793 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 25 May 2024 18:08:32 +0100 Subject: [PATCH 210/903] gh-99180: Make `StackSummary.should_show_carets` private (#119554) --- Lib/traceback.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/traceback.py b/Lib/traceback.py index 280d92d04cac9b..6ee1a50ca6804a 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -580,7 +580,7 @@ def format_frame_summary(self, frame_summary, **kwargs): show_carets = False with suppress(Exception): anchors = _extract_caret_anchors_from_line_segment(segment) - show_carets = self.should_show_carets(start_offset, end_offset, all_lines, anchors) + show_carets = self._should_show_carets(start_offset, end_offset, all_lines, anchors) result = [] @@ -694,7 +694,7 @@ def output_line(lineno): return ''.join(row) - def should_show_carets(self, start_offset, end_offset, all_lines, anchors): + def _should_show_carets(self, start_offset, end_offset, all_lines, anchors): with suppress(SyntaxError, ImportError): import ast tree = ast.parse('\n'.join(all_lines)) From 0c5ebe13e9937c446e9947c44f2570737ecca135 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 25 May 2024 15:30:48 -0400 Subject: [PATCH 211/903] gh-119560: Drop an Invalid Assert in PyState_FindModule() (gh-119561) The assertion was added in gh-118532 but was based on the invalid assumption that PyState_FindModule() would only be called with an already-initialized module def. I've added a test to make sure we don't make that assumption again. --- Lib/test/test_import/__init__.py | 7 ++ ...-05-25-12-52-25.gh-issue-119560.wSlm8q.rst | 3 + Modules/_testsinglephase.c | 91 ++++++++++++++++++- Python/import.c | 3 +- 4 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 64282d0f2d0bcf..11eaae5e47e97a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2887,6 +2887,13 @@ def test_with_reinit_reloaded(self): self.assertIs(reloaded.snapshot.cached, reloaded.module) + def test_check_state_first(self): + for variant in ['', '_with_reinit', '_with_state']: + name = f'{self.NAME}{variant}_check_cache_first' + with self.subTest(name): + mod = self._load_dynamic(name, self.ORIGIN) + self.assertEqual(mod.__name__, name) + # Currently, for every single-phrase init module loaded # in multiple interpreters, those interpreters share a # PyModuleDef for that object, which can be a problem. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst new file mode 100644 index 00000000000000..3a28a94df0f7cf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst @@ -0,0 +1,3 @@ +An invalid assert in beta 1 has been removed. The assert would fail if +``PyState_FindModule()`` was used in an extension module's init function +before the module def had been initialized. diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 448be502466e79..bcdb5ba31842fd 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -1,7 +1,7 @@ /* Testing module for single-phase initialization of extension modules -This file contains 5 distinct modules, meaning each as its own name +This file contains 8 distinct modules, meaning each as its own name and its own init function (PyInit_...). The default import system will only find the one matching the filename: _testsinglephase. To load the others you must do so manually. For example: @@ -14,7 +14,7 @@ spec = importlib.util.spec_from_file_location(name, filename, loader=loader) mod = importlib._bootstrap._load(spec) ``` -Here are the 5 modules: +Here are the 8 modules: * _testsinglephase * def: _testsinglephase_basic, @@ -136,6 +136,32 @@ Here are the 5 modules: 5. increment .initialized_count * functions: see common functions below * import system: same as _testsinglephase_basic_copy +* _testsinglephase_check_cache_first + * def: _testsinglepahse_check_cache_first + * m_name: "_testsinglephase_check_cache_first" + * m_size: -1 + * state: none + * init function: + * tries PyState_FindModule() first + * otherwise creates empty module + * functions: none + * import system: same as _testsinglephase +* _testsinglephase_with_reinit_check_cache_first + * def: _testsinglepahse_with_reinit_check_cache_first + * m_name: "_testsinglephase_with_reinit_check_cache_first" + * m_size: 0 + * state: none + * init function: same as _testsinglephase_check_cache_first + * functions: none + * import system: same as _testsinglephase_with_reinit +* _testsinglephase_with_state_check_cache_first + * def: _testsinglepahse_with_state_check_cache_first + * m_name: "_testsinglephase_with_state_check_cache_first" + * m_size: 42 + * state: none + * init function: same as _testsinglephase_check_cache_first + * functions: none + * import system: same as _testsinglephase_with_state Module state: @@ -650,3 +676,64 @@ PyInit__testsinglephase_with_state(void) finally: return module; } + + +/****************************************************/ +/* the _testsinglephase_*_check_cache_first modules */ +/****************************************************/ + +static struct PyModuleDef _testsinglephase_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_check_cache_first"), + .m_size = -1, // no module state +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_check_cache_first(void) +{ + assert(_testsinglephase_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_check_cache_first); +} + + +static struct PyModuleDef _testsinglephase_with_reinit_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_reinit_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_reinit_check_cache_first"), + .m_size = 0, // no module state +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_reinit_check_cache_first(void) +{ + assert(_testsinglephase_with_reinit_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_with_reinit_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_with_reinit_check_cache_first); +} + + +static struct PyModuleDef _testsinglephase_with_state_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_state_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_state_check_cache_first"), + .m_size = 42, // not used +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_state_check_cache_first(void) +{ + assert(_testsinglephase_with_state_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_with_state_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_with_state_check_cache_first); +} diff --git a/Python/import.c b/Python/import.c index ba44477318d473..4f3325aa67bd0a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -457,7 +457,6 @@ static Py_ssize_t _get_module_index_from_def(PyModuleDef *def) { Py_ssize_t index = def->m_base.m_index; - assert(index > 0); #ifndef NDEBUG struct extensions_cache_value *cached = _find_cached_def(def); assert(cached == NULL || index == _get_cached_module_index(cached)); @@ -489,7 +488,7 @@ _set_module_index(PyModuleDef *def, Py_ssize_t index) static const char * _modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) { - if (index == 0) { + if (index <= 0) { return "invalid module index"; } if (MODULES_BY_INDEX(interp) == NULL) { From e418fc3a6e7bade68ab5dfe72f14ddba28e6acb5 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 25 May 2024 21:01:36 +0100 Subject: [PATCH 212/903] GH-82805: Fix handling of single-dot file extensions in pathlib (#118952) pathlib now treats "`.`" as a valid file extension (suffix). This brings it in line with `os.path.splitext()`. In the (private) pathlib ABCs, we add a new `ParserBase.splitext()` method that splits a path into a `(root, ext)` pair, like `os.path.splitext()`. This method is called by `PurePathBase.stem`, `suffix`, etc. In a future version of pathlib, we might make these base classes public, and so users will be able to define their own `splitext()` method to control file extension splitting. In `pathlib.PurePath` we add optimised `stem`, `suffix` and `suffixes` properties that don't use `splitext()`, which avoids computing the path base name twice. --- Doc/library/pathlib.rst | 13 +++++ Lib/pathlib/_abc.py | 34 ++++++------- Lib/pathlib/_local.py | 34 +++++++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 50 ++++++++++++------- ...4-05-11-20-23-45.gh-issue-82805.F9bz4J.rst | 5 ++ 5 files changed, 101 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 71e2e5452d1754..c72d409a8eb2d6 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -449,6 +449,10 @@ Pure paths provide the following methods and properties: This is commonly called the file extension. + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. attribute:: PurePath.suffixes A list of the path's suffixes, often called file extensions:: @@ -460,6 +464,10 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').suffixes [] + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. attribute:: PurePath.stem @@ -713,6 +721,11 @@ Pure paths provide the following methods and properties: >>> p.with_suffix('') PureWindowsPath('README') + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. In previous + versions, :exc:`ValueError` is raised if a single dot is supplied. + .. method:: PurePath.with_segments(*pathsegments) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 3cdbb735096edb..6b5d9fc2a0c560 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -68,6 +68,12 @@ def splitdrive(self, path): drive. Either part may be empty.""" raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) + def splitext(self, path): + """Split the path into a pair (root, ext), where *ext* is empty or + begins with a begins with a period and contains at most one period, + and *root* is everything before the extension.""" + raise UnsupportedOperation(self._unsupported_msg('splitext()')) + def normcase(self, path): """Normalize the case of the path.""" raise UnsupportedOperation(self._unsupported_msg('normcase()')) @@ -151,12 +157,7 @@ def suffix(self): This includes the leading period. For example: '.txt' """ - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[i:] - else: - return '' + return self.parser.splitext(self.name)[1] @property def suffixes(self): @@ -165,21 +166,18 @@ def suffixes(self): These include the leading periods. For example: ['.tar', '.gz'] """ - name = self.name - if name.endswith('.'): - return [] - name = name.lstrip('.') - return ['.' + suffix for suffix in name.split('.')[1:]] + split = self.parser.splitext + stem, suffix = split(self.name) + suffixes = [] + while suffix: + suffixes.append(suffix) + stem, suffix = split(stem) + return suffixes[::-1] @property def stem(self): """The final path component, minus its last suffix.""" - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[:i] - else: - return name + return self.parser.splitext(self.name)[0] def with_name(self, name): """Return a new path with the file name changed.""" @@ -208,7 +206,7 @@ def with_suffix(self, suffix): if not stem: # If the stem is empty, we can't make the suffix non-empty. raise ValueError(f"{self!r} has an empty name") - elif suffix and not (suffix.startswith('.') and len(suffix) > 1): + elif suffix and not suffix.startswith('.'): raise ValueError(f"Invalid suffix {suffix!r}") else: return self.with_name(stem + suffix) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index f2776b1d20a2ea..49d9f813c54c23 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -361,6 +361,40 @@ def with_name(self, name): tail[-1] = name return self._from_parsed_parts(self.drive, self.root, tail) + @property + def stem(self): + """The final path component, minus its last suffix.""" + name = self.name + i = name.rfind('.') + if i != -1: + stem = name[:i] + # Stem must contain at least one non-dot character. + if stem.lstrip('.'): + return stem + return name + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + name = self.name.lstrip('.') + i = name.rfind('.') + if i != -1: + return name[i:] + return '' + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + return ['.' + ext for ext in self.name.lstrip('.').split('.')[1:]] + def relative_to(self, other, *, walk_up=False): """Return the relative path to another path identified by the passed arguments. If the operation is not possible (because this is not diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index d9e51c0e3d6411..57cc1612c03468 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -50,6 +50,7 @@ def test_unsupported_operation(self): self.assertRaises(e, m.join, 'foo') self.assertRaises(e, m.split, 'foo') self.assertRaises(e, m.splitdrive, 'foo') + self.assertRaises(e, m.splitext, 'foo') self.assertRaises(e, m.normcase, 'foo') self.assertRaises(e, m.isabs, 'foo') @@ -789,8 +790,12 @@ def test_suffix_common(self): self.assertEqual(P('/a/.hg.rc').suffix, '.rc') self.assertEqual(P('a/b.tar.gz').suffix, '.gz') self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('a/trailing.dot.').suffix, '.') + self.assertEqual(P('/a/trailing.dot.').suffix, '.') + self.assertEqual(P('a/..d.o.t..').suffix, '.') + self.assertEqual(P('a/inn.er..dots').suffix, '.dots') + self.assertEqual(P('photo').suffix, '') + self.assertEqual(P('photo.jpg').suffix, '.jpg') @needs_windows def test_suffix_windows(self): @@ -807,8 +812,8 @@ def test_suffix_windows(self): self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('c:a/trailing.dot.').suffix, '.') + self.assertEqual(P('c:/a/trailing.dot.').suffix, '.') self.assertEqual(P('//My.py/Share.php').suffix, '') self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') @@ -828,8 +833,12 @@ def test_suffixes_common(self): self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('/a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('a/..d.o.t..').suffixes, ['.o', '.t', '.', '.']) + self.assertEqual(P('a/inn.er..dots').suffixes, ['.er', '.', '.dots']) + self.assertEqual(P('photo').suffixes, []) + self.assertEqual(P('photo.jpg').suffixes, ['.jpg']) @needs_windows def test_suffixes_windows(self): @@ -848,8 +857,8 @@ def test_suffixes_windows(self): self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) self.assertEqual(P('//My.py/Share.php').suffixes, []) self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('c:a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('c:/a/trailing.dot.').suffixes, ['.dot', '.']) def test_stem_empty(self): P = self.cls @@ -865,8 +874,11 @@ def test_stem_common(self): self.assertEqual(P('a/.hgrc').stem, '.hgrc') self.assertEqual(P('a/.hg.rc').stem, '.hg') self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') + self.assertEqual(P('a/trailing.dot.').stem, 'trailing.dot') + self.assertEqual(P('a/..d.o.t..').stem, '..d.o.t.') + self.assertEqual(P('a/inn.er..dots').stem, 'inn.er.') + self.assertEqual(P('photo').stem, 'photo') + self.assertEqual(P('photo.jpg').stem, 'photo') @needs_windows def test_stem_windows(self): @@ -880,8 +892,8 @@ def test_stem_windows(self): self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') self.assertEqual(P('c:a/.hg.rc').stem, '.hg') self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') + self.assertEqual(P('c:a/trailing.dot.').stem, 'trailing.dot') + def test_with_name_common(self): P = self.cls self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) @@ -929,16 +941,16 @@ def test_with_stem_common(self): self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) - self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) + self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d.')) + self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d.')) @needs_windows def test_with_stem_windows(self): P = self.cls self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) - self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) + self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d.')) + self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d.')) self.assertRaises(ValueError, P('c:').with_stem, 'd') self.assertRaises(ValueError, P('c:/').with_stem, 'd') self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') @@ -974,6 +986,11 @@ def test_with_suffix_common(self): # Stripping suffix. self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + # Single dot + self.assertEqual(P('a/b').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b').with_suffix('.'), P('/a/b.')) + self.assertEqual(P('a/b.py').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b.py').with_suffix('.'), P('/a/b.')) @needs_windows def test_with_suffix_windows(self): @@ -1012,7 +1029,6 @@ def test_with_suffix_invalid(self): # Invalid suffix. self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') self.assertRaises(ValueError, P('a/b').with_suffix, '/') - self.assertRaises(ValueError, P('a/b').with_suffix, '.') self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') diff --git a/Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst b/Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst new file mode 100644 index 00000000000000..8715deda7d9c41 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst @@ -0,0 +1,5 @@ +Support single-dot file extensions in :attr:`pathlib.PurePath.suffix` and +related attributes and methods. For example, the +:attr:`~pathlib.PurePath.suffixes` of ``PurePath('foo.bar.')`` are now +``['.bar', '.']`` rather than ``[]``. This brings file extension splitting +in line with :func:`os.path.splitext`. From d25954dff5409c8926d2a4053d3e892462f8b8b5 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sat, 25 May 2024 21:13:31 -0400 Subject: [PATCH 213/903] docs: fix a few typos identified by codespell (#119516) --- Doc/c-api/weakref.rst | 4 ++-- Doc/extending/extending.rst | 2 +- Doc/extending/newtypes.rst | 2 +- Doc/howto/mro.rst | 2 +- Doc/library/curses.rst | 2 +- Doc/library/numbers.rst | 4 ++-- Doc/library/optparse.rst | 2 +- Doc/library/ssl.rst | 4 ++-- Doc/library/textwrap.rst | 2 +- Doc/library/turtle.rst | 2 +- Doc/tutorial/venv.rst | 2 +- Doc/using/ios.rst | 4 ++-- Doc/whatsnew/2.2.rst | 2 +- Doc/whatsnew/2.7.rst | 2 +- Doc/whatsnew/3.12.rst | 2 +- Doc/whatsnew/3.4.rst | 2 +- Doc/whatsnew/3.6.rst | 4 ++-- Doc/whatsnew/3.9.rst | 2 +- 18 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 038f54a9751fd1..ae0699383900c4 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -35,7 +35,7 @@ as much as it can. callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a - weakly referencable object, or if *callback* is not callable, ``None``, or + weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. @@ -47,7 +47,7 @@ as much as it can. be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* - is not a weakly referencable object, or if *callback* is not callable, + is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index b70e1b1fe57e67..b0493bed75b151 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -868,7 +868,7 @@ It is important to call :c:func:`free` at the right time. If a block's address is forgotten but :c:func:`free` is not called for it, the memory it occupies cannot be reused until the program terminates. This is called a :dfn:`memory leak`. On the other hand, if a program calls :c:func:`free` for a block and then -continues to use the block, it creates a conflict with re-use of the block +continues to use the block, it creates a conflict with reuse of the block through another :c:func:`malloc` call. This is called :dfn:`using freed memory`. It has the same bad consequences as referencing uninitialized data --- core dumps, wrong results, mysterious crashes. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 473a418809cff1..fd05c82b41629a 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -545,7 +545,7 @@ performance-critical objects (such as numbers). .. seealso:: Documentation for the :mod:`weakref` module. -For an object to be weakly referencable, the extension type must set the +For an object to be weakly referenceable, the extension type must set the ``Py_TPFLAGS_MANAGED_WEAKREF`` bit of the :c:member:`~PyTypeObject.tp_flags` field. The legacy :c:member:`~PyTypeObject.tp_weaklistoffset` field should be left as zero. diff --git a/Doc/howto/mro.rst b/Doc/howto/mro.rst index a44ef6848af4f3..f44b4f98e570bd 100644 --- a/Doc/howto/mro.rst +++ b/Doc/howto/mro.rst @@ -426,7 +426,7 @@ In this case the MRO is GFEF and the local precedence ordering is preserved. As a general rule, hierarchies such as the previous one should be -avoided, since it is unclear if F should override E or viceversa. +avoided, since it is unclear if F should override E or vice-versa. Python 2.3 solves the ambiguity by raising an exception in the creation of class G, effectively stopping the programmer from generating ambiguous hierarchies. The reason for that is that the C3 algorithm diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 883150e91378cc..91ea6150fb15ba 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -924,7 +924,7 @@ the following methods and attributes: .. method:: window.getbegyx() - Return a tuple ``(y, x)`` of co-ordinates of upper-left corner. + Return a tuple ``(y, x)`` of coordinates of upper-left corner. .. method:: window.getbkgd() diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 5f59746fa59812..d0ae79c7a3df76 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -84,10 +84,10 @@ The numeric tower ``~``. -Notes for type implementors +Notes for type implementers --------------------------- -Implementors should be careful to make equal numbers equal and hash +Implementers should be careful to make equal numbers equal and hash them to the same values. This may be subtle if there are two different extensions of the real numbers. For example, :class:`fractions.Fraction` implements :func:`hash` as follows:: diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 8c7d77d369b44c..3e96259f94d47b 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -1739,7 +1739,7 @@ seen, but blow up if it comes after ``-b`` in the command-line. :: Callback example 3: check option order (generalized) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you want to re-use this callback for several similar options (set a flag, but +If you want to reuse this callback for several similar options (set a flag, but blow up if ``-b`` has already been seen), it needs a bit of work: the error message and the flag that it sets must be generalized. :: diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 9c757ce1b8efc4..99abf45469018e 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -781,7 +781,7 @@ Constants .. data:: OP_SINGLE_DH_USE - Prevents re-use of the same DH key for distinct SSL sessions. This + Prevents reuse of the same DH key for distinct SSL sessions. This improves forward secrecy but requires more computational resources. This option only applies to server sockets. @@ -789,7 +789,7 @@ Constants .. data:: OP_SINGLE_ECDH_USE - Prevents re-use of the same ECDH key for distinct SSL sessions. This + Prevents reuse of the same ECDH key for distinct SSL sessions. This improves forward secrecy but requires more computational resources. This option only applies to server sockets. diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index deaefeee7b8c99..a58b460fef409c 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -154,7 +154,7 @@ hyphenated words; only then will long words be broken if necessary, unless wrapper = TextWrapper() wrapper.initial_indent = "* " - You can re-use the same :class:`TextWrapper` object many times, and you can + You can reuse the same :class:`TextWrapper` object many times, and you can change any of its options through direct assignment to instance attributes between uses. diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 2941201332a715..afda3685d606bb 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -120,7 +120,7 @@ off-screen):: home() The home position is at the center of the turtle's screen. If you ever need to -know them, get the turtle's x-y co-ordinates with:: +know them, get the turtle's x-y coordinates with:: pos() diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index 6cca3f1b25aadc..91e4ce18acef1d 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -38,7 +38,7 @@ Creating Virtual Environments The module used to create and manage virtual environments is called :mod:`venv`. :mod:`venv` will install the Python version from which the command was run (as reported by the :option:`--version` option). -For instance, excuting the command with ``python3.12`` will install +For instance, executing the command with ``python3.12`` will install version 3.12. To create a virtual environment, decide upon a directory where you want to diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index da8f42048c0faf..71fc29c450c8eb 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -303,8 +303,8 @@ modules in your app, some additional steps will be required: * You need to ensure that any folders containing third-party binaries are either associated with the app target, or copied in as part of step 8. Step 8 should also purge any binaries that are not appropriate for the platform a - specific build is targetting (i.e., delete any device binaries if you're - building app app targeting the simulator). + specific build is targeting (i.e., delete any device binaries if you're + building an app targeting the simulator). * Any folders that contain third-party binaries must be processed into framework form by step 9. The invocation of ``install_dylib`` that processes diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index e6c13f957b8d54..d4dbe0570fbda5 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1062,7 +1062,7 @@ code, none of the changes described here will affect you very much. simply been changed to use the new C-level interface. (Contributed by Fred L. Drake, Jr.) -* Another low-level API, primarily of interest to implementors of Python +* Another low-level API, primarily of interest to implementers of Python debuggers and development tools, was added. :c:func:`PyInterpreterState_Head` and :c:func:`PyInterpreterState_Next` let a caller walk through all the existing interpreter objects; :c:func:`PyInterpreterState_ThreadHead` and diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 031777b9cf6413..c45f0887b41f4f 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1738,7 +1738,7 @@ New module: importlib Python 3.1 includes the :mod:`importlib` package, a re-implementation of the logic underlying Python's :keyword:`import` statement. -:mod:`importlib` is useful for implementors of Python interpreters and +:mod:`importlib` is useful for implementers of Python interpreters and to users who wish to write new importers that can participate in the import process. Python 2.7 doesn't contain the complete :mod:`importlib` package, but instead has a tiny subset that contains diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index b64e4e205fe8c1..f99489fb53db74 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1251,7 +1251,7 @@ Deprecated :exc:`DeprecationWarning` when it can detect being called from a multithreaded process. There has always been a fundamental incompatibility with the POSIX platform when doing so. Even if such code *appeared* to work. - We added the warning to to raise awareness as issues encounted by code doing + We added the warning to raise awareness as issues encountered by code doing this are becoming more frequent. See the :func:`os.fork` documentation for more details along with `this discussion on fork being incompatible with threads `_ for *why* we're now surfacing this diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 3dd400c3771ed2..8aef0f5ac26728 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2413,7 +2413,7 @@ Changes in the Python API formal public interface the naming has been made consistent (:issue:`18532`). * Because :mod:`unittest.TestSuite` now drops references to tests after they - are run, test harnesses that re-use a :class:`~unittest.TestSuite` to re-run + are run, test harnesses that reuse a :class:`~unittest.TestSuite` to re-run a set of tests may fail. Test suites should not be re-used in this fashion since it means state is retained between test runs, breaking the test isolation that :mod:`unittest` is designed to provide. However, if the lack diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index d62beb0bdc8672..68ab43462b743a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2336,10 +2336,10 @@ Changes in the Python API * With the introduction of :exc:`ModuleNotFoundError`, import system consumers may start expecting import system replacements to raise that more specific exception when appropriate, rather than the less-specific :exc:`ImportError`. - To provide future compatibility with such consumers, implementors of + To provide future compatibility with such consumers, implementers of alternative import systems that completely replace :func:`__import__` will need to update their implementations to raise the new subclass when a module - can't be found at all. Implementors of compliant plugins to the default + can't be found at all. Implementers of compliant plugins to the default import system shouldn't need to make any changes, as the default import system will raise the new subclass when appropriate. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index e29d37ca120b76..90bdcf9541613c 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -891,7 +891,7 @@ Deprecated * Deprecated the ``split()`` method of :class:`!_tkinter.TkappType` in favour of the ``splitlist()`` method which has more consistent and - predicable behavior. + predictable behavior. (Contributed by Serhiy Storchaka in :issue:`38371`.) * The explicit passing of coroutine objects to :func:`asyncio.wait` has been From b5b7dc98c94100e992a5409d24bf035d88c7b2cd Mon Sep 17 00:00:00 2001 From: Wulian233 <71213467+Wulian233@users.noreply.github.com> Date: Sun, 26 May 2024 19:26:59 +0800 Subject: [PATCH 214/903] Update README and layout.html from 3.13 to 3.14 (#119539) Co-authored-by: Nice Zombies Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/tools/templates/layout.html | 8 +++++++- README.rst | 10 +++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index e931147813ae03..e96cbf70b1239e 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -61,7 +61,13 @@ // since all the "active, built and not hidden" versions will be shown automatically. let versions = config.versions.active.concat([ { - slug: "dev (3.13)", + slug: "dev (3.14)", + urls: { + documentation: "https://docs.python.org/3.14/", + } + }, + { + slug: "pre (3.13)", urls: { documentation: "https://docs.python.org/3.13/", } diff --git a/README.rst b/README.rst index 44b020e0c7ed5f..e3163c5ff636ab 100644 --- a/README.rst +++ b/README.rst @@ -136,7 +136,7 @@ What's New ---------- We have a comprehensive overview of the changes in the `What's New in Python -3.13 `_ document. For a more +3.14 `_ document. For a more detailed change log, read `Misc/NEWS `_, but a full accounting of changes can only be gleaned from the `commit history @@ -149,7 +149,7 @@ entitled "Installing multiple versions". Documentation ------------- -`Documentation for Python 3.13 `_ is online, +`Documentation for Python 3.14 `_ is online, updated daily. It can also be downloaded in many formats for faster access. The documentation @@ -200,15 +200,15 @@ intend to install multiple versions using the same prefix you must decide which version (if any) is your "primary" version. Install that version using ``make install``. Install all other versions using ``make altinstall``. -For example, if you want to install Python 2.7, 3.6, and 3.13 with 3.13 being the -primary version, you would execute ``make install`` in your 3.13 build directory +For example, if you want to install Python 2.7, 3.6, and 3.14 with 3.14 being the +primary version, you would execute ``make install`` in your 3.14 build directory and ``make altinstall`` in the others. Release Schedule ---------------- -See :pep:`719` for Python 3.13 release details. +See `PEP 745 `__ for Python 3.14 release details. Copyright and License Information From 008bc04dcb3b1fa6d7c11ed8050467dfad3090a9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 26 May 2024 13:34:48 +0100 Subject: [PATCH 215/903] gh-119562: Remove AST nodes deprecated since Python 3.8 (#119563) --- Doc/whatsnew/3.14.rst | 30 ++ Lib/ast.py | 174 +------- Lib/test/test_ast.py | 416 +----------------- ...-05-25-20-20-42.gh-issue-119562.DyplWc.rst | 3 + 4 files changed, 38 insertions(+), 585 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 39172ac60cf1e0..bc12d4b3b590dd 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -108,6 +108,36 @@ Deprecated Removed ======= +ast +--- + +* Remove the following classes. They were all deprecated since Python 3.8, + and have emitted deprecation warnings since Python 3.12: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. As a consequence of these removals, + user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, + ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom + :class:`ast.NodeVisitor` subclasses will no longer be called when the + ``NodeVisitor`` subclass is visiting an AST. Define a ``visit_Constant`` + method instead. + + Also, remove the following deprecated properties on :class:`ast.Constant`, + which were present for compatibility with the now-removed AST classes: + + * :attr:`!ast.Constant.n` + * :attr:`!ast.Constant.s` + + Use :attr:`!ast.Constant.value` instead. + + (Contributed by Alex Waygood in :gh:`119562`.) + + argparse -------- diff --git a/Lib/ast.py b/Lib/ast.py index 031bab43df7579..c5d495ea1c8000 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -508,27 +508,6 @@ def generic_visit(self, node): elif isinstance(value, AST): self.visit(value) - def visit_Constant(self, node): - value = node.value - type_name = _const_node_type_names.get(type(value)) - if type_name is None: - for cls, name in _const_node_type_names.items(): - if isinstance(value, cls): - type_name = name - break - if type_name is not None: - method = 'visit_' + type_name - try: - visitor = getattr(self, method) - except AttributeError: - pass - else: - import warnings - warnings.warn(f"{method} is deprecated; add visit_Constant", - DeprecationWarning, 2) - return visitor(node) - return self.generic_visit(node) - class NodeTransformer(NodeVisitor): """ @@ -597,142 +576,6 @@ def generic_visit(self, node): "use ast.Constant instead" ) - -# If the ast module is loaded more than once, only add deprecated methods once -if not hasattr(Constant, 'n'): - # The following code is for backward compatibility. - # It will be removed in future. - - def _n_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _n_setter(self, value): - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - def _s_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _s_setter(self, value): - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - Constant.n = property(_n_getter, _n_setter) - Constant.s = property(_s_getter, _s_setter) - -class _ABC(type): - - def __init__(cls, *args): - cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" - - def __instancecheck__(cls, inst): - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", - message=_DEPRECATED_CLASS_MESSAGE, - remove=(3, 14) - ) - if not isinstance(inst, Constant): - return False - if cls in _const_types: - try: - value = inst.value - except AttributeError: - return False - else: - return ( - isinstance(value, _const_types[cls]) and - not isinstance(value, _const_types_not.get(cls, ())) - ) - return type.__instancecheck__(cls, inst) - -def _new(cls, *args, **kwargs): - for key in kwargs: - if key not in cls._fields: - # arbitrary keyword arguments are accepted - continue - pos = cls._fields.index(key) - if pos < len(args): - raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(*args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -class Num(Constant, metaclass=_ABC): - _fields = ('n',) - __new__ = _new - -class Str(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class Bytes(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class NameConstant(Constant, metaclass=_ABC): - __new__ = _new - -class Ellipsis(Constant, metaclass=_ABC): - _fields = () - - def __new__(cls, *args, **kwargs): - if cls is _ast_Ellipsis: - import warnings - warnings._deprecated( - "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(..., *args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -# Keep another reference to Ellipsis in the global namespace -# so it can be referenced in Ellipsis.__new__ -# (The original "Ellipsis" name is removed from the global namespace later on) -_ast_Ellipsis = Ellipsis - -_const_types = { - Num: (int, float, complex), - Str: (str,), - Bytes: (bytes,), - NameConstant: (type(None), bool), - Ellipsis: (type(...),), -} -_const_types_not = { - Num: (bool,), -} - -_const_node_type_names = { - bool: 'NameConstant', # should be before int - type(None): 'NameConstant', - int: 'Num', - float: 'Num', - complex: 'Num', - str: 'Str', - bytes: 'Bytes', - type(...): 'Ellipsis', -} - class slice(AST): """Deprecated AST node class.""" @@ -1884,27 +1727,12 @@ def visit_MatchOr(self, node): self.set_precedence(_Precedence.BOR.next(), *node.patterns) self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) + def unparse(ast_obj): unparser = _Unparser() return unparser.visit(ast_obj) -_deprecated_globals = { - name: globals().pop(name) - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') -} - -def __getattr__(name): - if name in _deprecated_globals: - globals()[name] = value = _deprecated_globals[name] - import warnings - warnings._deprecated( - f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return value - raise AttributeError(f"module 'ast' has no attribute '{name}'") - - def main(): import argparse diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 8a4374c56cbc08..18b2f7ffca6083 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -8,9 +8,7 @@ import textwrap import types import unittest -import warnings import weakref -from functools import partial from textwrap import dedent try: import _testinternalcapi @@ -18,7 +16,6 @@ _testinternalcapi = None from test import support -from test.support.import_helper import import_fresh_module from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin @@ -223,7 +220,7 @@ def to_tuple(t): # These are compiled through "eval" # It should test all expressions eval_tests = [ - # None + # Constant(value=None) "None", # BoolOp "a and b", @@ -269,9 +266,9 @@ def to_tuple(t): "f(*[0, 1])", # Call with a generator argument "f(a for a in b)", - # Num + # Constant(value=int()) "10", - # Str + # Constant(value=str()) "'string'", # Attribute "a.b", @@ -498,35 +495,8 @@ def test_base_classes(self): self.assertTrue(issubclass(ast.comprehension, ast.AST)) self.assertTrue(issubclass(ast.Gt, ast.AST)) - def test_import_deprecated(self): - ast = import_fresh_module('ast') - depr_regex = ( - r'ast\.{} is deprecated and will be removed in Python 3.14; ' - r'use ast\.Constant instead' - ) - for name in 'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis': - with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): - getattr(ast, name) - - def test_field_attr_existence_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'): - item = getattr(ast, name) - if self._is_ast_node(name, item): - with self.subTest(item): - with self.assertWarns(DeprecationWarning): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - def test_field_attr_existence(self): for name, item in ast.__dict__.items(): - # These emit DeprecationWarnings - if name in {'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'}: - continue # constructor has a different signature if name == 'Index': continue @@ -569,106 +539,12 @@ def test_arguments(self): self.assertEqual(x.args, 2) self.assertEqual(x.vararg, 3) - def test_field_attr_writable_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - x = ast.Num() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - def test_field_attr_writable(self): x = ast.Constant(1) # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) - def test_classattrs_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - x = ast.Num() - self.assertEqual(x._fields, ('value', 'kind')) - - with self.assertRaises(AttributeError): - x.value - - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Num(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Num(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value', 'kind')) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) - - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - "Constant.__init__ missing 1 required positional argument: 'value'. This will become " - 'an error in Python 3.15.', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - "Constant.__init__ missing 1 required positional argument: 'value'. This will become " - 'an error in Python 3.15.', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - "Constant.__init__ got an unexpected keyword argument 'foo'. Support for " - 'arbitrary keyword arguments is deprecated and will be removed in Python ' - '3.15.', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - def test_classattrs(self): with self.assertWarns(DeprecationWarning): x = ast.Constant() @@ -714,190 +590,6 @@ def test_classattrs(self): self.assertIs(ast.Constant(None).value, None) self.assertIs(ast.Constant(...).value, ...) - def test_realtype(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.assertIs(type(ast.Num(42)), ast.Constant) - self.assertIs(type(ast.Num(4.25)), ast.Constant) - self.assertIs(type(ast.Num(4.25j)), ast.Constant) - self.assertIs(type(ast.Str('42')), ast.Constant) - self.assertIs(type(ast.Bytes(b'42')), ast.Constant) - self.assertIs(type(ast.NameConstant(True)), ast.Constant) - self.assertIs(type(ast.NameConstant(False)), ast.Constant) - self.assertIs(type(ast.NameConstant(None)), ast.Constant) - self.assertIs(type(ast.Ellipsis()), ast.Constant) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - def test_isinstance(self): - from ast import Constant - - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - cls_depr_msg = ( - 'ast.{} is deprecated and will be removed in Python 3.14; ' - 'use ast.Constant instead' - ) - - assertNumDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") - ) - assertStrDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") - ) - assertBytesDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") - ) - assertNameConstantDeprecated = partial( - self.assertWarnsRegex, - DeprecationWarning, - cls_depr_msg.format("NameConstant") - ) - assertEllipsisDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") - ) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - n = Num(arg) - with assertNumDeprecated(): - self.assertIsInstance(n, Num) - - with assertStrDeprecated(): - s = Str('42') - with assertStrDeprecated(): - self.assertIsInstance(s, Str) - - with assertBytesDeprecated(): - b = Bytes(b'42') - with assertBytesDeprecated(): - self.assertIsInstance(b, Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - n = NameConstant(arg) - with assertNameConstantDeprecated(): - self.assertIsInstance(n, NameConstant) - - with assertEllipsisDeprecated(): - e = Ellipsis() - with assertEllipsisDeprecated(): - self.assertIsInstance(e, Ellipsis) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertIsInstance(Constant(arg), Num) - - with assertStrDeprecated(): - self.assertIsInstance(Constant('42'), Str) - - with assertBytesDeprecated(): - self.assertIsInstance(Constant(b'42'), Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - self.assertIsInstance(Constant(arg), NameConstant) - - with assertEllipsisDeprecated(): - self.assertIsInstance(Constant(...), Ellipsis) - - with assertStrDeprecated(): - s = Str('42') - assertNumDeprecated(self.assertNotIsInstance, s, Num) - assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) - - with assertNumDeprecated(): - n = Num(42) - assertStrDeprecated(self.assertNotIsInstance, n, Str) - assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) - - with assertNameConstantDeprecated(): - n = NameConstant(True) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - with assertNameConstantDeprecated(): - n = NameConstant(False) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - for arg in '42', True, False: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(arg), Num) - - assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(1), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis) - - class S(str): pass - with assertStrDeprecated(): - self.assertIsInstance(Constant(S('42')), Str) - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(S('42')), Num) - - def test_constant_subclasses_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class N(ast.Num): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = 'spam' - class N2(ast.Num): - pass - - n = N(42) - self.assertEqual(n.n, 42) - self.assertEqual(n.z, 'spam') - self.assertIs(type(n), N) - self.assertIsInstance(n, N) - self.assertIsInstance(n, ast.Num) - self.assertNotIsInstance(n, N2) - self.assertNotIsInstance(ast.Num(42), N) - n = N(n=42) - self.assertEqual(n.n, 42) - self.assertIs(type(n), N) - - self.assertEqual([str(w.message) for w in wlog], [ - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - ]) - def test_constant_subclasses(self): class N(ast.Constant): def __init__(self, *args, **kwargs): @@ -2223,32 +1915,6 @@ def test_call(self): call = ast.Call(func, args, bad_keywords) self.expr(call, "must have Load context") - def test_num(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") @@ -2288,19 +1954,6 @@ def test_list(self): def test_tuple(self): self._sequence(ast.Tuple) - def test_nameconstant(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import NameConstant - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.expr(ast.NameConstant(4)) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - @support.requires_resource('cpu') def test_stdlib_validates(self): for module in STDLIB_FILES: @@ -2953,69 +2606,8 @@ def test_source_segment_missing_info(self): self.assertIsNone(ast.get_source_segment(s, x)) self.assertIsNone(ast.get_source_segment(s, y)) -class BaseNodeVisitorCases: - # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: - def test_old_constant_nodes(self): - class Visitor(self.visitor_class): - def visit_Num(self, node): - log.append((node.lineno, 'Num', node.n)) - def visit_Str(self, node): - log.append((node.lineno, 'Str', node.s)) - def visit_Bytes(self, node): - log.append((node.lineno, 'Bytes', node.s)) - def visit_NameConstant(self, node): - log.append((node.lineno, 'NameConstant', node.value)) - def visit_Ellipsis(self, node): - log.append((node.lineno, 'Ellipsis', ...)) - mod = ast.parse(dedent('''\ - i = 42 - f = 4.25 - c = 4.25j - s = 'string' - b = b'bytes' - t = True - n = None - e = ... - ''')) - visitor = Visitor() - log = [] - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - visitor.visit(mod) - self.assertEqual(log, [ - (1, 'Num', 42), - (2, 'Num', 4.25), - (3, 'Num', 4.25j), - (4, 'Str', 'string'), - (5, 'Bytes', b'bytes'), - (6, 'NameConstant', True), - (7, 'NameConstant', None), - (8, 'Ellipsis', ...), - ]) - self.assertEqual([str(w.message) for w in wlog], [ - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Str is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Bytes is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_Ellipsis is deprecated; add visit_Constant', - ]) - - -class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeVisitor - - -class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeTransformer +class NodeTransformerTests(ASTTestMixin, unittest.TestCase): def assertASTTransformation(self, tranformer_class, initial_code, expected_code): initial_ast = ast.parse(dedent(initial_code)) diff --git a/Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst b/Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst new file mode 100644 index 00000000000000..dd23466b9d2cef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst @@ -0,0 +1,3 @@ +Remove :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, +:class:`!ast.NameConstant` and :class:`!ast.Ellipsis`. They had all emitted +deprecation warnings since Python 3.12. Patch by Alex Waygood. From 70b07aa4153c1a914a3d69307d5b258cf7ed16ab Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 26 May 2024 14:37:33 +0200 Subject: [PATCH 216/903] gh-111997: Fix argument count for LINE event and clarify type of argument counts. (#119179) --- Python/instrumentation.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9095fb981b7981..a5211ee5428cf8 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -893,7 +893,7 @@ remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) static int call_one_instrument( PyInterpreterState *interp, PyThreadState *tstate, PyObject **args, - Py_ssize_t nargsf, int8_t tool, int event) + size_t nargsf, int8_t tool, int event) { assert(0 <= tool && tool < 8); assert(tstate->tracing == 0); @@ -1084,7 +1084,7 @@ call_instrumentation_vector( args[2] = offset_obj; PyInterpreterState *interp = tstate->interp; uint8_t tools = get_tools_for_instruction(code, interp, offset, event); - Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; PyObject **callargs = &args[1]; int err = 0; while (tools) { @@ -2439,13 +2439,15 @@ capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t PyErr_SetString(PyExc_ValueError, "offset must be non-negative"); return -1; } - PyObject *offset_obj = PyLong_FromLong(offset); - if (offset_obj == NULL) { - return -1; + if (event != PY_MONITORING_EVENT_LINE) { + PyObject *offset_obj = PyLong_FromLong(offset); + if (offset_obj == NULL) { + return -1; + } + assert(args[2] == NULL); + args[2] = offset_obj; } - assert(args[2] == NULL); - args[2] = offset_obj; - Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; PyObject **callargs = &args[1]; int err = 0; @@ -2565,8 +2567,8 @@ _PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_ if (lno == NULL) { return -1; } - PyObject *args[4] = { NULL, NULL, NULL, lno }; - int res= capi_call_instrumentation(state, codelike, offset, args, 3, + PyObject *args[3] = { NULL, NULL, lno }; + int res= capi_call_instrumentation(state, codelike, offset, args, 2, PY_MONITORING_EVENT_LINE); Py_DECREF(lno); return res; From 0220663e26aa2a5322df092078c5a16cddcc5cf4 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 26 May 2024 14:31:02 +0100 Subject: [PATCH 217/903] gh-119562: Remove unused private string constants from `ast.py` (#119576) --- Lib/ast.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index c5d495ea1c8000..bc6c3347787d61 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -567,15 +567,6 @@ def generic_visit(self, node): setattr(node, field, new_node) return node - -_DEPRECATED_VALUE_ALIAS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; use value instead" -) -_DEPRECATED_CLASS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; " - "use ast.Constant instead" -) - class slice(AST): """Deprecated AST node class.""" From 5d04cc50e51cb262ee189a6ef0e79f4b372d1583 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 26 May 2024 10:05:23 -0700 Subject: [PATCH 218/903] gh-102864: Add switching frame test for pdb (#119564) --- Lib/test/test_pdb.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 1b329b205d2d0f..cf69bc415c9b69 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2555,7 +2555,7 @@ def test_pdb_issue_gh_94215(): def test_pdb_issue_gh_101673(): """See GH-101673 - Make sure ll won't revert local variable assignment + Make sure ll and switching frames won't revert local variable assignment >>> def test_function(): ... a = 1 @@ -2565,6 +2565,10 @@ def test_pdb_issue_gh_101673(): ... '!a = 2', ... 'll', ... 'p a', + ... 'u', + ... 'p a', + ... 'd', + ... 'p a', ... 'continue' ... ]): ... test_function() @@ -2577,6 +2581,16 @@ def test_pdb_issue_gh_101673(): 3 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) p a 2 + (Pdb) u + > (11)() + -> test_function() + (Pdb) p a + *** NameError: name 'a' is not defined + (Pdb) d + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) p a + 2 (Pdb) continue """ From 5482a939ac18f4cd861d212c759960af8fa2b19d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 May 2024 21:33:16 -0400 Subject: [PATCH 219/903] Re-order imports to align with zipp 3.18.2 (#119587) --- Lib/test/test_zipfile/_path/test_complexity.py | 2 +- Lib/test/test_zipfile/_path/test_path.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py index fd7ce57551b7a5..b505dd7c376462 100644 --- a/Lib/test/test_zipfile/_path/test_complexity.py +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -20,7 +20,7 @@ class TestComplexity(unittest.TestCase): @pytest.mark.flaky def test_implied_dirs_performance(self): best, others = big_o.big_o( - compose(consume, zipfile.CompleteDirs._implied_dirs), + compose(consume, zipfile._path.CompleteDirs._implied_dirs), lambda size: [ '/'.join(string.ascii_lowercase + str(n)) for n in range(size) ], diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index f6e2c8c289f6fd..e5d2acf39a10f8 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -8,13 +8,13 @@ import zipfile import zipfile._path +from test.support.os_helper import temp_dir, FakePath + from ._functools import compose from ._itertools import Counter from ._test_params import parameterize, Invoked -from test.support.os_helper import temp_dir, FakePath - class jaraco: class itertools: From 5ef5622543844bad1f9bc770ddaaddd2615b8466 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Mon, 27 May 2024 15:57:23 +0800 Subject: [PATCH 220/903] Fix typos in HISTORY documentation (#119453) --- Misc/ACKS | 1 + Misc/HISTORY | 70 ++++++++++++++++++++++++++-------------------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS index eaa7453aaade3e..9c10a76f1df624 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -2054,6 +2054,7 @@ Doug Wyatt Xiang Zhang Robert Xiao Florent Xicluna +Yanbo, Xie Xinhang Xu Arnon Yaari Alakshendra Yadav diff --git a/Misc/HISTORY b/Misc/HISTORY index b66413277259dc..8ca35e1af62c05 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -607,7 +607,7 @@ Library MemoryError. - Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed - ambigious reverse mappings. Added many new mappings. Import mapping is no + ambiguous reverse mappings. Added many new mappings. Import mapping is no longer applied to modules already mapped with full name mapping. - Issue #23745: The new email header parser now handles duplicate MIME @@ -2030,7 +2030,7 @@ Library initialization of the unquote_to_bytes() table of the urllib.parse module, to not waste memory if these modules are not used. -- Issue #19157: Include the broadcast address in the usuable hosts for IPv6 +- Issue #19157: Include the broadcast address in the usable hosts for IPv6 in ipaddress. - Issue #11599: When an external command (e.g. compiler) fails, distutils now @@ -2620,7 +2620,7 @@ Library - asyncio: Various improvements and small changes not all covered by issues listed below. E.g. wait_for() now cancels the inner task if - the timeout occcurs; tweaked the set of exported symbols; renamed + the timeout occurs; tweaked the set of exported symbols; renamed Empty/Full to QueueEmpty/QueueFull; "with (yield from lock)" now uses a separate context manager; readexactly() raises if not enough data was read; PTY support tweaks. @@ -3944,7 +3944,7 @@ Library - Issue #18996: TestCase.assertEqual() now more cleverly shorten differing strings in error report. -- Issue #19034: repr() for tkinter.Tcl_Obj now exposes string reperesentation. +- Issue #19034: repr() for tkinter.Tcl_Obj now exposes string representation. - Issue #18978: ``urllib.request.Request`` now allows the method to be indicated on the class and no longer sets it to None in ``__init__``. @@ -4191,7 +4191,7 @@ Library - Issue #18532: Change the builtin hash algorithms' names to lower case names as promised by hashlib's documentation. -- Issue #8713: add new spwan and forkserver start methods, and new functions +- Issue #8713: add new spawn and forkserver start methods, and new functions get_all_start_methods, get_start_method, and set_start_method, to multiprocessing. @@ -4524,7 +4524,7 @@ Core and Builtins - Issue #16613: Add *m* argument to ``collections.Chainmap.new_child`` to allow the new child map to be specified explicitly. -- Issue #16730: importlib.machinery.FileFinder now no longers raises an +- Issue #16730: importlib.machinery.FileFinder now no longer raises an exception when trying to populate its cache and it finds out the directory is unreadable or has turned into a file. Reported and diagnosed by David Pritchard. @@ -4832,7 +4832,7 @@ Library on Windows and adds no value over and above python -m pydoc ... - Issue #18155: The csv module now correctly handles csv files that use - a delimter character that has a special meaning in regexes, instead of + a delimiter character that has a special meaning in regexes, instead of throwing an exception. - Issue #14360: encode_quopri can now be successfully used as an encoder @@ -6329,7 +6329,7 @@ Documentation - Issue #15940: Specify effect of locale on time functions. -- Issue #17538: Document XML vulnerabilties +- Issue #17538: Document XML vulnerabilities - Issue #16642: sched.scheduler timefunc initial default is time.monotonic. Patch by Ramchandra Apte @@ -6676,7 +6676,7 @@ Library - Issue #14669: Fix pickling of connections and sockets on Mac OS X by sending/receiving an acknowledgment after file descriptor transfer. - TestPicklingConnection has been reenabled for Mac OS X. + TestPicklingConnection has been re-enabled for Mac OS X. - Issue #11062: Fix adding a message from file to Babyl mailbox. @@ -7114,7 +7114,7 @@ Build - Issue #14330: For cross builds, don't use host python, use host search paths for host compiler. -- Issue #15235: Allow Berkley DB versions up to 5.3 to build the dbm module. +- Issue #15235: Allow Berkeley DB versions up to 5.3 to build the dbm module. - Issue #15268: Search curses.h in /usr/include/ncursesw. @@ -7264,7 +7264,7 @@ Library called with no arguments. - Issue #14653: email.utils.mktime_tz() no longer relies on system - mktime() when timezone offest is supplied. + mktime() when timezone offset is supplied. - Issue #14684: zlib.compressobj() and zlib.decompressobj() now support the use of predefined compression dictionaries. Original patch by Sam Rushing. @@ -7606,7 +7606,7 @@ Library - Issue #14773: Fix os.fwalk() failing on dangling symlinks. - Issue #12541: Be lenient with quotes around Realm field of HTTP Basic - Authentation in urllib2. + Authentication in urllib2. - Issue #14807: move undocumented tarfile.filemode() to stat.filemode() and add doc entry. Add tarfile.filemode alias with deprecation warning. @@ -7673,7 +7673,7 @@ Library IDLE ---- -- Issue #14958: Change IDLE systax highlighting to recognize all string and +- Issue #14958: Change IDLE syntax highlighting to recognize all string and byte literals supported in Python 3.3. - Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. @@ -10176,7 +10176,7 @@ IDLE - Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. (Patch by Roger Serwy) -- Issue #9871: Prevent IDLE 3 crash when given byte stings +- Issue #9871: Prevent IDLE 3 crash when given byte strings with invalid hex escape sequences, like b'\x0'. (Original patch by Claudiu Popa.) @@ -12098,7 +12098,7 @@ Library - Issue #9632: Remove sys.setfilesystemencoding() function: use PYTHONFSENCODING environment variable to set the filesystem encoding at Python startup. sys.setfilesystemencoding() creates inconsistencies because it is unable to - reencode all filenames in all objects. + re-encode all filenames in all objects. - Issue #9410: Various optimizations to the pickle module, leading to speedups up to 4x (depending on the benchmark). Mostly ported from Unladen Swallow; @@ -12509,7 +12509,7 @@ Library - Issue #9605: posix.getlogin() decodes the username with file filesystem encoding and surrogateescape error handler. Patch written by David Watson. -- Issue #9604: posix.initgroups() encodes the username using the fileystem +- Issue #9604: posix.initgroups() encodes the username using the filesystem encoding and surrogateescape error handler. Patch written by David Watson. - Issue #9603: posix.ttyname() and posix.ctermid() decode the terminal name @@ -12667,7 +12667,7 @@ What's New in Python 3.2 Alpha 1? Core and Builtins ----------------- -- Issue #8991: convertbuffer() rejects discontigious buffers. +- Issue #8991: convertbuffer() rejects discontiguous buffers. - Issue #7616: Fix copying of overlapping memoryview slices with the Intel compiler. @@ -13211,7 +13211,7 @@ Library - Issue #7989: Added pure python implementation of the `datetime` module. The C module is renamed to `_datetime` and if available, overrides all classes - defined in datetime with fast C impementation. Python implementation is based + defined in datetime with fast C implementation. Python implementation is based on the original python prototype for the datetime module by Tim Peters with minor modifications by the PyPy project. The test suite now tests `datetime` module with and without `_datetime` acceleration using the same test cases. @@ -15049,7 +15049,7 @@ Extension Modules an error. The _PY_STRUCT_FLOAT_COERCE constant has been removed. The version number has been bumped to 0.3. -- Issue #5359: Readd the Berkeley DB detection code to allow _dbm be built +- Issue #5359: Re-add the Berkeley DB detection code to allow _dbm be built using Berkeley DB. Tests @@ -17028,7 +17028,7 @@ Extension Modules and renamed to filter(), map(), and zip(). Also, renamed izip_longest() to zip_longest() and ifilterfalse() to filterfalse(). -- Issue #1762972: Readded the reload() function as imp.reload(). +- Issue #1762972: Re-added the reload() function as imp.reload(). - Bug #2111: mmap segfaults when trying to write a block opened with PROT_READ. @@ -18448,7 +18448,7 @@ Core and builtins - Fixed bug #1459029 - unicode reprs were double-escaped. -- Patch #1396919: The system scope threads are reenabled on FreeBSD +- Patch #1396919: The system scope threads are re-enabled on FreeBSD 5.4 and later versions. - Bug #1115379: Compiling a Unicode string with an encoding declaration @@ -21803,7 +21803,7 @@ Library - New csv package makes it easy to read/write CSV files. - Module shlex has been extended to allow posix-like shell parsings, - including a split() function for easy spliting of quoted strings and + including a split() function for easy splitting of quoted strings and commands. An iterator interface was also implemented. Tools/Demos @@ -27751,7 +27751,7 @@ Fri Mar 12 22:15:43 1999 Guido van Rossum The filename to URL conversion didn't properly quote special characters. - The URL to filename didn't properly unquote special chatacters. + The URL to filename didn't properly unquote special characters. * Objects/floatobject.c: OK, try again. Vladimir gave me a fix for the alignment bus error, @@ -27807,7 +27807,7 @@ Wed Mar 10 22:55:47 1999 Guido van Rossum classes in selected module methods of selected class - Sinlge clicking in a directory, module or class item updates the next + Single clicking in a directory, module or class item updates the next column with info about the selected item. Double clicking in a module, class or method item opens the file (and selects the clicked item if it is a class or method). @@ -28130,7 +28130,7 @@ webchecker and other ftp retrieves. - ConfigParser's get() method now accepts an optional keyword argument (vars) that is substituted on top of the defaults that were setup in -__init__. You can now also have recusive references in your +__init__. You can now also have recursive references in your configuration file. - Some improvements to the Queue module, including a put_nowait() @@ -28209,7 +28209,7 @@ core. not. - The curses module implements an optional nlines argument to -w.scroll(). (It then calls wscrl(win, nlines) instead of scoll(win).) +w.scroll(). (It then calls wscrl(win, nlines) instead of scroll(win).) Changes to tools ---------------- @@ -28504,7 +28504,7 @@ PyEval_GetGlobals. - glmodule.c: check in the changed version after running the stubber again -- this solves the conflict with curses over the 'clear' entry point much nicer. (Jack Jansen had checked in the changes to cstubs -eons ago, but I never regenrated glmodule.c :-( ) +eons ago, but I never regenerated glmodule.c :-( ) - frameobject.c: fix reference count bug in PyFrame_New. Vladimir Marangozov. @@ -28581,7 +28581,7 @@ idiom L1[len(L1):] = L2. - Better error messages when a sequence is indexed with a non-integer. -- Bettter error message when calling a non-callable object (include +- Better error message when calling a non-callable object (include the type in the message). Python services @@ -28656,7 +28656,7 @@ Internet Protocols and Support - imaplib.py: new version from Piers Lauder. - smtplib.py: change sendmail() method to accept a single string or a -list or strings as the destination (commom newbie mistake). +list or strings as the destination (common newbie mistake). - poplib.py: LIST with a msg argument fixed. @@ -31109,7 +31109,7 @@ encoding/decoding CGI form arguments. Catch all errors from the ftp module. HTTP requests now add the Host: header line. The proxy variable names are now mapped to lower case, for Windows. The spliturl() function no longer erroneously throws away all data past -the first newline. The basejoin() function now intereprets "../" +the first newline. The basejoin() function now interprets "../" correctly. I *believe* that the problems with "exception raised in __del__" under certain circumstances have been fixed (mostly by changes elsewher in the interpreter). @@ -31397,7 +31397,7 @@ changes and fixes. - Added a bunch of new winfo options to Tkinter.py; we should now be up to date with Tk 4.2. The new winfo options supported are: -mananger, pointerx, pointerxy, pointery, server, viewable, visualid, +manager, pointerx, pointerxy, pointery, server, viewable, visualid, visualsavailable. - The broken bind() method on Canvas objects defined in the Canvas.py @@ -32552,7 +32552,7 @@ The same applies to posixfile.open() and the socket method makefile(). is being maintained and distributed separately. - Improved support for the Apple Macintosh, in part by Jack Jansen, -e.g. interfaces to (a few) resource mananger functions, get/set file +e.g. interfaces to (a few) resource manager functions, get/set file type and creator, gestalt, sound manager, speech manager, MacTCP, comm toolbox, and the think C console library. This is being maintained and distributed separately. @@ -33229,7 +33229,7 @@ sys.argv[0]; it can simply do "if __name__ == '__main__': main()". * When an object is printed by the print statement, its implementation of str() is used. This means that classes can define __str__(self) to direct how their instances are printed. This is different from -__repr__(self), which should define an unambigous string +__repr__(self), which should define an unambiguous string representation of the instance. (If __str__() is not defined, it defaults to __repr__().) @@ -34366,7 +34366,7 @@ eval_code) and ceval.h (which doesn't need compile.hand declares the rest) ceval.h defines macros BGN_SAVE / END_SAVE for use with threads (to -improve the parallellism of multi-threaded programs by letting other +improve the parallelism of multi-threaded programs by letting other Python code run when a blocking system call or something similar is made) @@ -34514,7 +34514,7 @@ names listed in a 'global' statement must not be used in the function before the statement is reached. Remember that you don't need to use 'global' if you only want to *use* -a global variable in a function; nor do you need ot for assignments to +a global variable in a function; nor do you need to for assignments to parts of global variables (e.g., list or dictionary items or attributes of class instances). This has not changed; in fact assignment to part of a global variable was the standard workaround. From c7a5e1e550a2a0bfa11dbf055ed4b7afb26b5fe9 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 27 May 2024 14:35:36 +0300 Subject: [PATCH 221/903] ``Include/internal/pycore_import.h``: Fix typo (#119586) Fix typo --- Include/internal/pycore_import.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index bd40707fed21a8..f8329a460d6cbf 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -50,7 +50,7 @@ struct _import_runtime_state { PyMutex mutex; /* The actual cache of (filename, name, PyModuleDef) for modules. Only legacy (single-phase init) extension modules are added - and only if they support multiple initialization (m_size >- 0) + and only if they support multiple initialization (m_size >= 0) or are imported in the main interpreter. This is initialized lazily in fix_up_extension() in import.c. Modules are added there and looked up in _imp.find_extension(). */ From 3b26cd8ca0e6c65e4b61effea9aa44d06e926797 Mon Sep 17 00:00:00 2001 From: Aditya Borikar Date: Mon, 27 May 2024 06:16:13 -0600 Subject: [PATCH 222/903] gh-119467: Fix Py_buffer.format type and correct documentation typo (#119475) --- Doc/c-api/buffer.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 1e1cabdf242bd1..9500fe465c7d94 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -147,9 +147,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - .. c:member:: const char *format + .. c:member:: char *format - A *NUL* terminated string in :mod:`struct` module style syntax describing + A *NULL* terminated string in :mod:`struct` module style syntax describing the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes) is assumed. From 041a566f3f987619cef7d6ae7915ba93e39d2d1e Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 27 May 2024 05:20:28 -0700 Subject: [PATCH 223/903] GH-117283: Add doc warning for `PyTuple_SetItem` refcount > 1 (#117916) --- Doc/c-api/tuple.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 0d68a360f347f8..52668d16b74436 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -105,6 +105,12 @@ Tuple Objects is being replaced; any reference in the tuple at position *pos* will be leaked. + .. warning:: + + This macro should *only* be used on tuples that are newly created. + Using this macro on a tuple that is already in use (or in other words, has + a refcount > 1) could lead to undefined behavior. + .. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) From 59630f92d8223f80993e3646b0f734d27f4b8dd4 Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Mon, 27 May 2024 09:39:59 -0300 Subject: [PATCH 224/903] Docs: Add class role for IPV{4,6}Address and fix a typo (#118059) Add class role for IPV{4,6}Address and fix a typo Co-authored-by: Kumar Aditya --- Doc/library/ipaddress.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index d7dccf1a86593d..ead841b0581e21 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -504,7 +504,7 @@ dictionaries. 4. A two-tuple of an address description and a netmask, where the address description is either a string, a 32-bits integer, a 4-bytes packed - integer, or an existing IPv4Address object; and the netmask is either + integer, or an existing :class:`IPv4Address` object; and the netmask is either an integer representing the prefix length (e.g. ``24``) or a string representing the prefix mask (e.g. ``255.255.255.0``). @@ -725,7 +725,7 @@ dictionaries. 4. A two-tuple of an address description and a netmask, where the address description is either a string, a 128-bits integer, a 16-bytes packed - integer, or an existing IPv6Address object; and the netmask is an + integer, or an existing :class:`IPv6Address` object; and the netmask is an integer representing the prefix length. An :exc:`AddressValueError` is raised if *address* is not a valid IPv6 @@ -781,7 +781,7 @@ dictionaries. .. attribute:: is_site_local - These attribute is true for the network as a whole if it is true + This attribute is true for the network as a whole if it is true for both the network address and the broadcast address. From 88e3fee3f81f3470cf4fe2e2611441071779e884 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 27 May 2024 20:29:27 +0300 Subject: [PATCH 225/903] Docs: Only install sphinx-autobuild for `make htmllive` (#119607) --- Doc/Makefile | 6 +++++- Doc/requirements.txt | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/Makefile b/Doc/Makefile index eca574ec290af7..1cbfc722b010f5 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -150,10 +150,14 @@ gettext: build htmlview: html $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" +.PHONY: ensure-sphinx-autobuild +ensure-sphinx-autobuild: venv + $(VENVDIR)/bin/sphinx-autobuild --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install sphinx-autobuild + .PHONY: htmllive htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 -htmllive: html +htmllive: ensure-sphinx-autobuild html .PHONY: clean clean: clean-venv diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 15675ab45fea71..b47a9d8a8635ab 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -10,7 +10,6 @@ sphinx~=7.3.0 blurb -sphinx-autobuild sphinxext-opengraph==0.7.5 sphinx-notfound-page==1.0.0 From 3dfa364cf2ae94e797b25fe5cac74b016a5a7fe6 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 27 May 2024 10:54:23 -0700 Subject: [PATCH 226/903] gh-119580: Improve version added section for convenience variable (#119583) --- Doc/library/pdb.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 7d67e06434b799..cd6496203949ea 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -290,6 +290,8 @@ There are three preset *convenience variables*: .. versionadded:: 3.12 + Added the *convenience variable* feature. + .. index:: pair: .pdbrc; file triple: debugger; configuration; file From eea26c4a731ff9547d48a6761b209fee3f2f84df Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 27 May 2024 21:04:34 +0300 Subject: [PATCH 227/903] Docs: Move inline JavaScript to own file to reduce duplication (#119541) --- Doc/tools/static/rtd_switcher.js | 88 ++++++++++++++++++++++++++++++ Doc/tools/templates/layout.html | 93 +------------------------------- 2 files changed, 90 insertions(+), 91 deletions(-) create mode 100644 Doc/tools/static/rtd_switcher.js diff --git a/Doc/tools/static/rtd_switcher.js b/Doc/tools/static/rtd_switcher.js new file mode 100644 index 00000000000000..a67bb85505a9ca --- /dev/null +++ b/Doc/tools/static/rtd_switcher.js @@ -0,0 +1,88 @@ + function onSwitch(event) { + const option = event.target.selectedIndex; + const item = event.target.options[option]; + window.location.href = item.dataset.url; + } + + document.addEventListener("readthedocs-addons-data-ready", function(event) { + const config = event.detail.data() + + // Add some mocked hardcoded versions pointing to the official + // documentation while migrating to Read the Docs. + // These are only for testing purposes. + // TODO: remove them when managing all the versions on Read the Docs, + // since all the "active, built and not hidden" versions will be shown automatically. + let versions = config.versions.active.concat([ + { + slug: "dev (3.14)", + urls: { + documentation: "https://docs.python.org/3.14/", + } + }, + { + slug: "dev (3.13)", + urls: { + documentation: "https://docs.python.org/3.13/", + } + }, + { + slug: "3.12", + urls: { + documentation: "https://docs.python.org/3.12/", + } + }, + { + slug: "3.11", + urls: { + documentation: "https://docs.python.org/3.11/", + } + }, + ]); + + const versionSelect = ` + + `; + + // Prepend the current language to the options on the selector + let languages = config.projects.translations.concat(config.projects.current); + languages = languages.sort((a, b) => a.language.name.localeCompare(b.language.name)); + + const languageSelect = ` + + `; + + // Query all the placeholders because there are different ones for Desktop/Mobile + const versionPlaceholders = document.querySelectorAll(".version_switcher_placeholder"); + for (placeholder of versionPlaceholders) { + placeholder.innerHTML = versionSelect; + let selectElement = placeholder.querySelector("select"); + selectElement.addEventListener("change", onSwitch); + } + + const languagePlaceholders = document.querySelectorAll(".language_switcher_placeholder"); + for (placeholder of languagePlaceholders) { + placeholder.innerHTML = languageSelect; + let selectElement = placeholder.querySelector("select"); + selectElement.addEventListener("change", onSwitch); + } + }); diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index e96cbf70b1239e..3f88fc8e91faad 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -43,96 +43,7 @@ {{ super() }} {%- if not embedded %} - - + + {%- endif %} {% endblock %} From 3ff06ebec4e8b466f76078aa9c97cea2093d52ab Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 May 2024 11:07:16 -0700 Subject: [PATCH 228/903] Withdraw most of my ownership in favor of Mark (#119611) --- .github/CODEOWNERS | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e955567ec0b0f8..e08d6cc5719737 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -29,20 +29,20 @@ Objects/type* @markshannon Objects/codeobject.c @markshannon Objects/frameobject.c @markshannon Objects/call.c @markshannon -Python/ceval*.c @markshannon @gvanrossum -Python/ceval*.h @markshannon @gvanrossum +Python/ceval*.c @markshannon +Python/ceval*.h @markshannon Python/compile.c @markshannon @iritkatriel Python/assemble.c @markshannon @iritkatriel Python/flowgraph.c @markshannon @iritkatriel Python/ast_opt.c @isidentical -Python/bytecodes.c @markshannon @gvanrossum -Python/optimizer*.c @markshannon @gvanrossum +Python/bytecodes.c @markshannon +Python/optimizer*.c @markshannon Python/optimizer_analysis.c @Fidget-Spinner Python/optimizer_bytecodes.c @Fidget-Spinner Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv Lib/test/test_patma.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra -Lib/test/test_capi/test_misc.py @markshannon @gvanrossum +Lib/test/test_capi/test_misc.py @markshannon Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv Tools/c-analyzer/ @ericsnowcurrently @@ -152,7 +152,7 @@ Include/internal/pycore_time.h @pganssle @abalkin /Lib/test/test_tokenize.py @pablogsal @lysnikolaou # Code generator -/Tools/cases_generator/ @gvanrossum +/Tools/cases_generator/ @markshannon # AST Python/ast.c @isidentical From 0bd0d4072a49df49a88e8b02c3258dbd294170f6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 27 May 2024 13:22:57 -0500 Subject: [PATCH 229/903] Misc cleanups and wording improvements for the itertools docs (gh-119626) --- Doc/library/itertools.rst | 238 +++++++++++++++++++------------------- 1 file changed, 116 insertions(+), 122 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 43432dae1623ce..121bfd3de343c4 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -56,13 +56,13 @@ Iterator Arguments Results :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` -:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) → 6 4 1`` -:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x%2, range(10)) → 0 2 4 6 8`` +:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` +:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` -:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) → 1 4`` +:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n :func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D-`` ============================ ============================ ================================================= ============================================================= @@ -97,31 +97,27 @@ The following module functions all construct and return iterators. Some provide streams of infinite length, so they should only be accessed by functions or loops that truncate the stream. -.. function:: accumulate(iterable[, func, *, initial=None]) - Make an iterator that returns accumulated sums, or accumulated - results of other binary functions (specified via the optional - *func* argument). +.. function:: accumulate(iterable[, function, *, initial=None]) - If *func* is supplied, it should be a function - of two arguments. Elements of the input *iterable* may be any type - that can be accepted as arguments to *func*. (For example, with - the default operation of addition, elements may be any addable - type including :class:`~decimal.Decimal` or - :class:`~fractions.Fraction`.) + Make an iterator that returns accumulated sums or accumulated + results from other binary functions. - Usually, the number of elements output matches the input iterable. - However, if the keyword argument *initial* is provided, the - accumulation leads off with the *initial* value so that the output - has one more element than the input iterable. + The *function* defaults to addition. The *function* should accept + two arguments, an accumulated total and a value from the *iterable*. + + If an *initial* value is provided, the accumulation will start with + that value and the output will have one more element than the input + iterable. Roughly equivalent to:: - def accumulate(iterable, func=operator.add, *, initial=None): + def accumulate(iterable, function=operator.add, *, initial=None): 'Return running totals' # accumulate([1,2,3,4,5]) → 1 3 6 10 15 # accumulate([1,2,3,4,5], initial=100) → 100 101 103 106 110 115 # accumulate([1,2,3,4,5], operator.mul) → 1 2 6 24 120 + iterator = iter(iterable) total = initial if initial is None: @@ -129,27 +125,29 @@ loops that truncate the stream. total = next(iterator) except StopIteration: return + yield total for element in iterator: - total = func(total, element) + total = function(total, element) yield total - The *func* argument can be set to - :func:`min` for a running minimum, :func:`max` for a running maximum, or - :func:`operator.mul` for a running product. Amortization tables can be - built by accumulating interest and applying payments: + The *function* argument can be set to :func:`min` for a running + minimum, :func:`max` for a running maximum, or :func:`operator.mul` + for a running product. `Amortization tables + `_ + can be built by accumulating interest and applying payments: .. doctest:: >>> data = [3, 4, 6, 2, 1, 9, 0, 7, 5, 8] - >>> list(accumulate(data, operator.mul)) # running product - [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] >>> list(accumulate(data, max)) # running maximum [3, 4, 6, 6, 6, 9, 9, 9, 9, 9] + >>> list(accumulate(data, operator.mul)) # running product + [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] # Amortize a 5% loan of 1000 with 10 annual payments of 90 - >>> account_update = lambda bal, pmt: round(bal * 1.05) + pmt - >>> list(accumulate(repeat(-90, 10), account_update, initial=1_000)) + >>> update = lambda balance, payment: round(balance * 1.05) - payment + >>> list(accumulate(repeat(90, 10), update, initial=1_000)) [1000, 960, 918, 874, 828, 779, 728, 674, 618, 559, 497] See :func:`functools.reduce` for a similar function that returns only the @@ -158,7 +156,7 @@ loops that truncate the stream. .. versionadded:: 3.2 .. versionchanged:: 3.3 - Added the optional *func* parameter. + Added the optional *function* parameter. .. versionchanged:: 3.8 Added the optional *initial* parameter. @@ -190,8 +188,8 @@ loops that truncate the stream. # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') - iterable = iter(iterable) - while batch := tuple(islice(iterable, n)): + iterator = iter(iterable) + while batch := tuple(islice(iterator, n)): if strict and len(batch) != n: raise ValueError('batched(): incomplete batch') yield batch @@ -230,12 +228,17 @@ loops that truncate the stream. Return *r* length subsequences of elements from the input *iterable*. + The output is a subsequence of :func:`product` keeping only entries that + are subsequences of the *iterable*. The length of the output is given + by :func:`math.comb` which computes ``n! / r! / (n - r)!`` when ``0 ≤ r + ≤ n`` or zero when ``r > n``. + The combination tuples are emitted in lexicographic order according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + the order of the input *iterable*. If the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So, if the input elements are unique, there will be no repeated + value. If the input elements are unique, there will be no repeated values within each combination. Roughly equivalent to:: @@ -243,11 +246,13 @@ loops that truncate the stream. def combinations(iterable, r): # combinations('ABCD', 2) → AB AC AD BC BD CD # combinations(range(4), 3) → 012 013 023 123 + pool = tuple(iterable) n = len(pool) if r > n: return indices = list(range(r)) + yield tuple(pool[i] for i in indices) while True: for i in reversed(range(r)): @@ -260,42 +265,36 @@ loops that truncate the stream. indices[j] = indices[j-1] + 1 yield tuple(pool[i] for i in indices) - The code for :func:`combinations` can be also expressed as a subsequence - of :func:`permutations` after filtering entries where the elements are not - in sorted order (according to their position in the input pool):: - - def combinations(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in permutations(range(n), r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / r! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. .. function:: combinations_with_replacement(iterable, r) Return *r* length subsequences of elements from the input *iterable* allowing individual elements to be repeated more than once. + The output is a subsequence of :func:`product` that keeps only entries + that are subsequences (with possible repeated elements) of the + *iterable*. The number of subsequence returned is ``(n + r - 1)! / r! / + (n - 1)!`` when ``n > 0``. + The combination tuples are emitted in lexicographic order according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + the order of the input *iterable*. if the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So, if the input elements are unique, the generated combinations + value. If the input elements are unique, the generated combinations will also be unique. Roughly equivalent to:: def combinations_with_replacement(iterable, r): # combinations_with_replacement('ABC', 2) → AA AB AC BB BC CC + pool = tuple(iterable) n = len(pool) if not n and r: return indices = [0] * r + yield tuple(pool[i] for i in indices) while True: for i in reversed(range(r)): @@ -306,28 +305,15 @@ loops that truncate the stream. indices[i:] = [indices[i] + 1] * (r - i) yield tuple(pool[i] for i in indices) - The code for :func:`combinations_with_replacement` can be also expressed as - a subsequence of :func:`product` after filtering entries where the elements - are not in sorted order (according to their position in the input pool):: - - def combinations_with_replacement(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in product(range(n), repeat=r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``(n+r-1)! / r! / (n-1)!`` when ``n > 0``. - .. versionadded:: 3.1 .. function:: compress(data, selectors) - Make an iterator that filters elements from *data* returning only those that - have a corresponding element in *selectors* is true. - Stops when either the *data* or *selectors* iterables have been exhausted. - Roughly equivalent to:: + Make an iterator that returns elements from *data* where the + corresponding element in *selectors* is true. Stops when either the + *data* or *selectors* iterables have been exhausted. Roughly + equivalent to:: def compress(data, selectors): # compress('ABCDEF', [1,0,1,0,1,1]) → A C E F @@ -338,9 +324,10 @@ loops that truncate the stream. .. function:: count(start=0, step=1) - Make an iterator that returns evenly spaced values starting with number *start*. Often - used as an argument to :func:`map` to generate consecutive data points. - Also, used with :func:`zip` to add sequence numbers. Roughly equivalent to:: + Make an iterator that returns evenly spaced values beginning with + *start*. Can be used with :func:`map` to generate consecutive data + points or with :func:`zip` to add sequence numbers. Roughly + equivalent to:: def count(start=0, step=1): # count(10) → 10 11 12 13 14 ... @@ -357,11 +344,12 @@ loops that truncate the stream. .. versionchanged:: 3.1 Added *step* argument and allowed non-integer arguments. + .. function:: cycle(iterable) - Make an iterator returning elements from the iterable and saving a copy of each. - When the iterable is exhausted, return elements from the saved copy. Repeats - indefinitely. Roughly equivalent to:: + Make an iterator returning elements from the *iterable* and saving a + copy of each. When the iterable is exhausted, return elements from + the saved copy. Repeats indefinitely. Roughly equivalent to:: def cycle(iterable): # cycle('ABCD') → A B C D A B C D A B C D ... @@ -373,32 +361,38 @@ loops that truncate the stream. for element in saved: yield element - Note, this member of the toolkit may require significant auxiliary storage - (depending on the length of the iterable). + This itertool may require significant auxiliary storage (depending on + the length of the iterable). .. function:: dropwhile(predicate, iterable) - Make an iterator that drops elements from the iterable as long as the predicate - is true; afterwards, returns every element. Note, the iterator does not produce - *any* output until the predicate first becomes false, so it may have a lengthy - start-up time. Roughly equivalent to:: + Make an iterator that drops elements from the *iterable* while the + *predicate* is true and afterwards returns every element. Roughly + equivalent to:: def dropwhile(predicate, iterable): # dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8 - iterable = iter(iterable) - for x in iterable: + + iterator = iter(iterable) + for x in iterator: if not predicate(x): yield x break - for x in iterable: + + for x in iterator: yield x + Note this does not produce *any* output until the predicate first + becomes false, so this itertool may have a lengthy start-up time. + + .. function:: filterfalse(predicate, iterable) - Make an iterator that filters elements from iterable returning only those for - which the predicate is false. If *predicate* is ``None``, return the items - that are false. Roughly equivalent to:: + Make an iterator that filters elements from the *iterable* returning + only those for which the *predicate* returns a false value. If + *predicate* is ``None``, returns the items that are false. Roughly + equivalent to:: def filterfalse(predicate, iterable): # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 @@ -473,20 +467,19 @@ loops that truncate the stream. .. function:: islice(iterable, stop) islice(iterable, start, stop[, step]) - Make an iterator that returns selected elements from the iterable. If *start* is - non-zero, then elements from the iterable are skipped until start is reached. - Afterward, elements are returned consecutively unless *step* is set higher than - one which results in items being skipped. If *stop* is ``None``, then iteration - continues until the iterator is exhausted, if at all; otherwise, it stops at the - specified position. + Make an iterator that returns selected elements from the iterable. + Works like sequence slicing but does not support negative values for + *start*, *stop*, or *step*. + + If *start* is zero or ``None``, iteration starts at zero. Otherwise, + elements from the iterable are skipped until *start* is reached. - If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, - then the step defaults to one. + If *stop* is ``None``, iteration continues until the iterator is + exhausted, if at all. Otherwise, it stops at the specified position. - Unlike regular slicing, :func:`islice` does not support negative values for - *start*, *stop*, or *step*. Can be used to extract related fields from - data where the internal structure has been flattened (for example, a - multi-line report may list a name field on every third line). + If *step* is ``None``, the step defaults to one. Elements are returned + consecutively unless *step* is set higher than one which results in + items being skipped. Roughly equivalent to:: @@ -534,18 +527,24 @@ loops that truncate the stream. .. function:: permutations(iterable, r=None) - Return successive *r* length permutations of elements in the *iterable*. + Return successive *r* length `permutations of elements + `_ from the *iterable*. If *r* is not specified or is ``None``, then *r* defaults to the length of the *iterable* and all possible full-length permutations are generated. + The output is a subsequence of :func:`product` where entries with + repeated elements have been filtered out. The length of the output is + given by :func:`math.perm` which computes ``n! / (n - r)!`` when + ``0 ≤ r ≤ n`` or zero when ``r > n``. + The permutation tuples are emitted in lexicographic order according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + the order of the input *iterable*. If the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So, if the input elements are unique, there will be no repeated + value. If the input elements are unique, there will be no repeated values within a permutation. Roughly equivalent to:: @@ -578,20 +577,6 @@ loops that truncate the stream. else: return - The code for :func:`permutations` can be also expressed as a subsequence of - :func:`product` filtered to exclude entries with repeated elements (those - from the same position in the input pool):: - - def permutations(iterable, r=None): - pool = tuple(iterable) - n = len(pool) - r = n if r is None else r - for indices in product(range(n), repeat=r): - if len(set(indices)) == r: - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. .. function:: product(*iterables, repeat=1) @@ -615,10 +600,13 @@ loops that truncate the stream. def product(*iterables, repeat=1): # product('ABCD', 'xy') → Ax Ay Bx By Cx Cy Dx Dy # product(range(2), repeat=3) → 000 001 010 011 100 101 110 111 + pools = [tuple(pool) for pool in iterables] * repeat + result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] + for prod in result: yield tuple(prod) @@ -626,6 +614,7 @@ loops that truncate the stream. keeping pools of values in memory to generate the products. Accordingly, it is only useful with finite inputs. + .. function:: repeat(object[, times]) Make an iterator that returns *object* over and over again. Runs indefinitely @@ -650,12 +639,12 @@ loops that truncate the stream. >>> list(map(pow, range(10), repeat(2))) [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] + .. function:: starmap(function, iterable) - Make an iterator that computes the function using arguments obtained from - the iterable. Used instead of :func:`map` when argument parameters are already - grouped in tuples from a single iterable (when the data has been - "pre-zipped"). + Make an iterator that computes the *function* using arguments obtained + from the *iterable*. Used instead of :func:`map` when argument + parameters have already been "pre-zipped" into tuples. The difference between :func:`map` and :func:`starmap` parallels the distinction between ``function(a,b)`` and ``function(*c)``. Roughly @@ -669,8 +658,8 @@ loops that truncate the stream. .. function:: takewhile(predicate, iterable) - Make an iterator that returns elements from the iterable as long as the - predicate is true. Roughly equivalent to:: + Make an iterator that returns elements from the *iterable* as long as + the *predicate* is true. Roughly equivalent to:: def takewhile(predicate, iterable): # takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4 @@ -726,9 +715,15 @@ loops that truncate the stream. .. function:: zip_longest(*iterables, fillvalue=None) - Make an iterator that aggregates elements from each of the iterables. If the - iterables are of uneven length, missing values are filled-in with *fillvalue*. - Iteration continues until the longest iterable is exhausted. Roughly equivalent to:: + Make an iterator that aggregates elements from each of the + *iterables*. + + If the iterables are of uneven length, missing values are filled-in + with *fillvalue*. If not specified, *fillvalue* defaults to ``None``. + + Iteration continues until the longest iterable is exhausted. + + Roughly equivalent to:: def zip_longest(*iterables, fillvalue=None): # zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D- @@ -754,8 +749,7 @@ loops that truncate the stream. If one of the iterables is potentially infinite, then the :func:`zip_longest` function should be wrapped with something that limits the number of calls - (for example :func:`islice` or :func:`takewhile`). If not specified, - *fillvalue* defaults to ``None``. + (for example :func:`islice` or :func:`takewhile`). .. _itertools-recipes: From ae7b17673f29efe17b416cbcfbf43b5b3ff5977c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 27 May 2024 15:35:30 -0400 Subject: [PATCH 230/903] gh-119584: Fix test_import Failed Assertion (gh-119623) The fix in gh-119561 introduced an assertion that doesn't hold true if any of the three new test extension modules are loaded more than once. This is fine normally but breaks if the new test_check_state_first() is run more than once, which happens for refleak checking and with the regrtest --forever flag. We fix that here by clearing each of the three modules after loading them. We also tweak a check in _modules_by_index_check(). --- Lib/test/test_import/__init__.py | 3 +++ Modules/_testsinglephase.c | 3 +++ Python/import.c | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 11eaae5e47e97a..b09065f812c0e4 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2887,12 +2887,15 @@ def test_with_reinit_reloaded(self): self.assertIs(reloaded.snapshot.cached, reloaded.module) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_check_state_first(self): for variant in ['', '_with_reinit', '_with_state']: name = f'{self.NAME}{variant}_check_cache_first' with self.subTest(name): mod = self._load_dynamic(name, self.ORIGIN) self.assertEqual(mod.__name__, name) + sys.modules.pop(name, None) + _testinternalcapi.clear_extension(name, self.ORIGIN) # Currently, for every single-phrase init module loaded # in multiple interpreters, those interpreters share a diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index bcdb5ba31842fd..066e0dbfb63fbf 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -682,6 +682,9 @@ PyInit__testsinglephase_with_state(void) /* the _testsinglephase_*_check_cache_first modules */ /****************************************************/ +/* Each of these modules should only be freshly loaded. That means + clearing the caches and each module def's m_base after each load. */ + static struct PyModuleDef _testsinglephase_check_cache_first = { PyModuleDef_HEAD_INIT, .m_name = "_testsinglephase_check_cache_first", diff --git a/Python/import.c b/Python/import.c index 4f3325aa67bd0a..6fe6df4db4f55e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -494,7 +494,7 @@ _modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) if (MODULES_BY_INDEX(interp) == NULL) { return "Interpreters module-list not accessible."; } - if (index > PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { + if (index >= PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { return "Module index out of bounds."; } return NULL; @@ -2183,7 +2183,7 @@ clear_singlephase_extension(PyInterpreterState *interp, /* Clear data set when the module was initially loaded. */ def->m_base.m_init = NULL; Py_CLEAR(def->m_base.m_copy); - // We leave m_index alone since there's no reason to reset it. + def->m_base.m_index = 0; /* Clear the PyState_*Module() cache entry. */ Py_ssize_t index = _get_cached_module_index(cached); From 3e8b60905e97a4fe89bb24180063732214368938 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 28 May 2024 00:02:46 +0200 Subject: [PATCH 231/903] gh-117398: Add multiphase support to _datetime (gh-119373) This is minimal support. Subinterpreters are not supported yet. That will be addressed in a later change. Co-authored-by: Eric Snow --- Lib/test/datetimetester.py | 21 +++++++++++++++++++++ Modules/_datetimemodule.c | 26 +++++++++++--------------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b3838d5b406e94..ba7f185a092629 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -47,6 +47,26 @@ pass # +# This is copied from test_import/__init__.py. +# XXX Move it to support/__init__.py. +def no_rerun(reason): + """Skip rerunning for a particular test. + + WARNING: Use this decorator with care; skipping rerunning makes it + impossible to find reference leaks. Provide a clear reason for skipping the + test using the 'reason' parameter. + """ + def deco(func): + _has_run = False + def wrapper(self): + nonlocal _has_run + if _has_run: + self.skipTest(reason) + func(self) + _has_run = True + return wrapper + return deco + pickle_loads = {pickle.loads, pickle._loads} pickle_choices = [(pickle, pickle, proto) @@ -6383,6 +6403,7 @@ class IranTest(ZoneInfoTest): @unittest.skipIf(_testcapi is None, 'need _testcapi module') +@no_rerun("the encapsulated datetime C API does not support reloading") class CapiTest(unittest.TestCase): def setUp(self): # Since the C API is not present in the _Pure tests, skip all tests diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 3c6d270b8d1331..271b37dfcded6c 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7040,30 +7040,26 @@ _datetime_exec(PyObject *module) } #undef DATETIME_ADD_MACRO -static struct PyModuleDef datetimemodule = { +static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _datetime_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static PyModuleDef datetimemodule = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_datetime", .m_doc = "Fast implementation of the datetime type.", - .m_size = -1, + .m_size = 0, .m_methods = module_methods, + .m_slots = module_slots, }; PyMODINIT_FUNC PyInit__datetime(void) { - PyObject *mod = PyModule_Create(&datetimemodule); - if (mod == NULL) - return NULL; -#ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); -#endif - - if (_datetime_exec(mod) < 0) { - Py_DECREF(mod); - return NULL; - } - - return mod; + return PyModuleDef_Init(&datetimemodule); } /* --------------------------------------------------------------------------- From a9a74da4a0ca0645f049e67b6434a95e30592c32 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 27 May 2024 19:50:38 -0700 Subject: [PATCH 232/903] gh-119311: Fix name mangling with PEP 695 generic classes (#119464) Fixes #119311. Fixes #119395. --- Include/internal/pycore_symtable.h | 2 + Lib/importlib/_bootstrap_external.py | 3 +- Lib/test/test_type_params.py | 94 +++++++++++++++++++ ...-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst | 2 + ...-05-23-06-34-45.gh-issue-119311.2DBwKR.rst | 2 + Python/compile.c | 12 +-- Python/symtable.c | 49 ++++++++-- 7 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 16e89f80d9d0c8..ac6c499c08264e 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -81,6 +81,7 @@ typedef struct _symtable_entry { PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ PyObject *ste_directives;/* locations of global and nonlocal statements */ + PyObject *ste_mangled_names; /* set of names for which mangling should be applied */ _Py_block_ty ste_type; int ste_nested; /* true if block is nested */ unsigned ste_free : 1; /* true if block has free variables */ @@ -128,6 +129,7 @@ extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); extern void _PySymtable_Free(struct symtable *); +extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name); extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); /* Flags for def-use information */ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 68469863e7f774..30c91801212374 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -473,6 +473,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a5 3569 (Specialize CONTAINS_OP) # Python 3.13a6 3570 (Add __firstlineno__ class attribute) # Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT) +# Python 3.14a1 3601 (Fix miscompilation of private names in generic classes) # Python 3.15 will start with 3700 @@ -489,7 +490,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3600).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3601).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 82f1007f9ac97b..aa9d4088bd7ccc 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -823,6 +823,100 @@ def meth[__U](self, arg: __T, arg2: __U): self.assertEqual(Foo.Alias.__value__, (T, V)) + def test_no_leaky_mangling_in_module(self): + ns = run_code(""" + __before = "before" + class X[T]: pass + __after = "after" + """) + self.assertEqual(ns["__before"], "before") + self.assertEqual(ns["__after"], "after") + + def test_no_leaky_mangling_in_function(self): + ns = run_code(""" + def f(): + class X[T]: pass + _X_foo = 2 + __foo = 1 + assert locals()['__foo'] == 1 + return __foo + """) + self.assertEqual(ns["f"](), 1) + + def test_no_leaky_mangling_in_class(self): + ns = run_code(""" + class Outer: + __before = "before" + class Inner[T]: + __x = "inner" + __after = "after" + """) + Outer = ns["Outer"] + self.assertEqual(Outer._Outer__before, "before") + self.assertEqual(Outer.Inner._Inner__x, "inner") + self.assertEqual(Outer._Outer__after, "after") + + def test_no_mangling_in_bases(self): + ns = run_code(""" + class __Base: + def __init_subclass__(self, **kwargs): + self.kwargs = kwargs + + class Derived[T](__Base, __kwarg=1): + pass + """) + Derived = ns["Derived"] + self.assertEqual(Derived.__bases__, (ns["__Base"], Generic)) + self.assertEqual(Derived.kwargs, {"__kwarg": 1}) + + def test_no_mangling_in_nested_scopes(self): + ns = run_code(""" + from test.test_type_params import make_base + + class __X: + pass + + class Y[T: __X]( + make_base(lambda: __X), + # doubly nested scope + make_base(lambda: (lambda: __X)), + # list comprehension + make_base([__X for _ in (1,)]), + # genexp + make_base(__X for _ in (1,)), + ): + pass + """) + Y = ns["Y"] + T, = Y.__type_params__ + self.assertIs(T.__bound__, ns["__X"]) + base0 = Y.__bases__[0] + self.assertIs(base0.__arg__(), ns["__X"]) + base1 = Y.__bases__[1] + self.assertIs(base1.__arg__()(), ns["__X"]) + base2 = Y.__bases__[2] + self.assertEqual(base2.__arg__, [ns["__X"]]) + base3 = Y.__bases__[3] + self.assertEqual(list(base3.__arg__), [ns["__X"]]) + + def test_type_params_are_mangled(self): + ns = run_code(""" + from test.test_type_params import make_base + + class Foo[__T, __U: __T](make_base(__T), make_base(lambda: __T)): + param = __T + """) + Foo = ns["Foo"] + T, U = Foo.__type_params__ + self.assertEqual(T.__name__, "__T") + self.assertEqual(U.__name__, "__U") + self.assertIs(U.__bound__, T) + self.assertIs(Foo.param, T) + + base1, base2, *_ = Foo.__bases__ + self.assertIs(base1.__arg__, T) + self.assertIs(base2.__arg__(), T) + class TypeParamsComplexCallsTest(unittest.TestCase): def test_defaults(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst new file mode 100644 index 00000000000000..24cd90a8e5e5db --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst @@ -0,0 +1,2 @@ +Fix bug where names appearing after a generic class are mangled as if they +are in the generic class. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst new file mode 100644 index 00000000000000..9e0db37340c49a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst @@ -0,0 +1,2 @@ +Fix bug where names are unexpectedly mangled in the bases of generic +classes. diff --git a/Python/compile.c b/Python/compile.c index 6dacfc7cb55aa6..cdc5b26ec70066 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1001,7 +1001,7 @@ static int compiler_addop_name(struct compiler_unit *u, location loc, int opcode, PyObject *dict, PyObject *o) { - PyObject *mangled = _Py_Mangle(u->u_private, o); + PyObject *mangled = _Py_MaybeMangle(u->u_private, u->u_ste, o); if (!mangled) { return ERROR; } @@ -1873,7 +1873,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc, arg_ty arg = asdl_seq_GET(kwonlyargs, i); expr_ty default_ = asdl_seq_GET(kw_defaults, i); if (default_) { - PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg); + PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, arg->arg); if (!mangled) { goto error; } @@ -1930,7 +1930,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id, if (!annotation) { return SUCCESS; } - PyObject *mangled = _Py_Mangle(c->u->u_private, id); + PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, id); if (!mangled) { return ERROR; } @@ -2625,7 +2625,6 @@ compiler_class(struct compiler *c, stmt_ty s) asdl_type_param_seq *type_params = s->v.ClassDef.type_params; int is_generic = asdl_seq_LEN(type_params) > 0; if (is_generic) { - Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); PyObject *type_params_name = PyUnicode_FromFormat("", s->v.ClassDef.name); if (!type_params_name) { @@ -2637,6 +2636,7 @@ compiler_class(struct compiler *c, stmt_ty s) return ERROR; } Py_DECREF(type_params_name); + Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); _Py_DECLARE_STR(type_params, ".type_params"); RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store)); @@ -4203,7 +4203,7 @@ compiler_nameop(struct compiler *c, location loc, return ERROR; } - mangled = _Py_Mangle(c->u->u_private, name); + mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, name); if (!mangled) { return ERROR; } @@ -6512,7 +6512,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) VISIT(c, expr, s->v.AnnAssign.annotation); } ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); - mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); + mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id); ADDOP_LOAD_CONST_NEW(c, loc, mangled); ADDOP(c, loc, STORE_SUBSCR); } diff --git a/Python/symtable.c b/Python/symtable.c index 2ec21a2d376da2..d8240cdd11f7ea 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -103,6 +103,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_children = NULL; ste->ste_directives = NULL; + ste->ste_mangled_names = NULL; ste->ste_type = block; ste->ste_nested = 0; @@ -166,6 +167,7 @@ ste_dealloc(PySTEntryObject *ste) Py_XDECREF(ste->ste_varnames); Py_XDECREF(ste->ste_children); Py_XDECREF(ste->ste_directives); + Py_XDECREF(ste->ste_mangled_names); PyObject_Free(ste); } @@ -1338,6 +1340,11 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, if (prev) { ste->ste_comp_iter_expr = prev->ste_comp_iter_expr; } + /* No need to inherit ste_mangled_names in classes, where all names + * are mangled. */ + if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) { + ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names); + } /* The entry is owned by the stack. Borrow it for st_cur. */ Py_DECREF(ste); st->st_cur = ste; @@ -1363,7 +1370,7 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, static long symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { - PyObject *mangled = _Py_Mangle(st->st_private, name); + PyObject *mangled = _Py_MaybeMangle(st->st_private, ste, name); if (!mangled) return 0; long ret = _PyST_GetSymbol(ste, mangled); @@ -1384,8 +1391,7 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s PyObject *o; PyObject *dict; long val; - PyObject *mangled = _Py_Mangle(st->st_private, name); - + PyObject *mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name); if (!mangled) return 0; @@ -1474,6 +1480,11 @@ static int symtable_add_def(struct symtable *st, PyObject *name, int flag, int lineno, int col_offset, int end_lineno, int end_col_offset) { + if ((flag & DEF_TYPE_PARAM) && st->st_cur->ste_mangled_names != NULL) { + if(PySet_Add(st->st_cur->ste_mangled_names, name) < 0) { + return 0; + } + } return symtable_add_def_helper(st, name, flag, st->st_cur, lineno, col_offset, end_lineno, end_col_offset); } @@ -1508,7 +1519,6 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, lineno, col_offset, end_lineno, end_col_offset)) { return 0; } - st->st_private = name; // This is used for setting the generic base _Py_DECLARE_STR(generic_base, ".generic_base"); if (!symtable_add_def(st, &_Py_STR(generic_base), DEF_LOCAL, @@ -1597,7 +1607,7 @@ symtable_record_directive(struct symtable *st, identifier name, int lineno, if (!st->st_cur->ste_directives) return 0; } - mangled = _Py_Mangle(st->st_private, name); + mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name); if (!mangled) return 0; data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset); @@ -1673,6 +1683,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); if (s->v.ClassDef.decorator_list) VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list); + tmp = st->st_private; if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_enter_type_param_block(st, s->v.ClassDef.name, (void *)s->v.ClassDef.type_params, @@ -1680,6 +1691,11 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) LOCATION(s))) { VISIT_QUIT(st, 0); } + st->st_private = s->v.ClassDef.name; + st->st_cur->ste_mangled_names = PySet_New(NULL); + if (!st->st_cur->ste_mangled_names) { + VISIT_QUIT(st, 0); + } VISIT_SEQ(st, type_param, s->v.ClassDef.type_params); } VISIT_SEQ(st, expr, s->v.ClassDef.bases); @@ -1688,7 +1704,6 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) (void *)s, s->lineno, s->col_offset, s->end_lineno, s->end_col_offset)) VISIT_QUIT(st, 0); - tmp = st->st_private; st->st_private = s->v.ClassDef.name; if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_add_def(st, &_Py_ID(__type_params__), @@ -1702,13 +1717,13 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } } VISIT_SEQ(st, stmt, s->v.ClassDef.body); - st->st_private = tmp; if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); } + st->st_private = tmp; break; } case TypeAlias_kind: { @@ -2776,6 +2791,26 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, return st; } +PyObject * +_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name) +{ + /* Special case for type parameter blocks around generic classes: + * we want to mangle type parameter names (so a type param with a private + * name can be used inside the class body), but we don't want to mangle + * any other names that appear within the type parameter scope. + */ + if (ste->ste_mangled_names != NULL) { + int result = PySet_Contains(ste->ste_mangled_names, name); + if (result < 0) { + return NULL; + } + if (result == 0) { + return Py_NewRef(name); + } + } + return _Py_Mangle(privateobj, name); +} + PyObject * _Py_Mangle(PyObject *privateobj, PyObject *ident) { From bf08f0a5fe5750904aa4a239945db16d2c43f6e7 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Tue, 28 May 2024 15:53:32 +0800 Subject: [PATCH 233/903] Fix typos in comments (#119645) --- Tools/clinic/libclinic/dsl_parser.py | 2 +- Tools/peg_generator/pegen/parser_generator.py | 2 +- Tools/wasm/wasi-env | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index cb18374cf07e3c..ab9b586693d01c 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -119,7 +119,7 @@ class ParamState(enum.IntEnum): # Legal transitions: to LEFT_SQUARE_BEFORE or REQUIRED START = 0 - # Left square backets before required params. + # Left square brackets before required params. LEFT_SQUARE_BEFORE = 1 # In a group, before required params. diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index ad28f6c27dcb37..3f386b61be5898 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -41,7 +41,7 @@ class RuleCollectorVisitor(GrammarVisitor): - """Visitor that invokes a provieded callmaker visitor with just the NamedItem nodes""" + """Visitor that invokes a provided callmaker visitor with just the NamedItem nodes""" def __init__(self, rules: Dict[str, Rule], callmakervisitor: GrammarVisitor) -> None: self.rulses = rules diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index e6c6fb2d8e47e7..95eda863cb62c6 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -71,5 +71,5 @@ export CFLAGS LDFLAGS export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR export PATH -# no exec, it makes arvg[0] path absolute. +# no exec, it makes argv[0] path absolute. "$@" From b313cc68d50de5fb5f43acffd402c5c4da6516fc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 28 May 2024 12:01:37 +0300 Subject: [PATCH 234/903] gh-117557: Improve error messages when a string, bytes or bytearray of length 1 are expected (GH-117631) --- Lib/test/clinic.test.c | 261 +++++++++++++++--- Lib/test/test_ctypes/test_functions.py | 26 +- Lib/test/test_ctypes/test_parameters.py | 22 +- Lib/test/test_str.py | 2 +- ...-04-08-13-49-02.gh-issue-117558.9lSEpR.rst | 2 + Modules/_ctypes/cfield.c | 59 ++-- Modules/_cursesmodule.c | 32 ++- Modules/arraymodule.c | 40 ++- Modules/clinic/_testclinic.c.h | 259 +++++++++++++++-- Modules/clinic/arraymodule.c.h | 7 +- Modules/clinic/unicodedata.c.h | 52 +++- Modules/termios.c | 2 +- Objects/bytesobject.c | 29 +- Objects/stringlib/clinic/transmogrify.h.h | 56 +++- Objects/unicodeobject.c | 16 +- PC/clinic/msvcrtmodule.c.h | 48 +++- Python/getargs.c | 34 ++- Tools/clinic/libclinic/converters.py | 25 +- 18 files changed, 811 insertions(+), 161 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-08-13-49-02.gh-issue-117558.9lSEpR.rst diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 58ffc0ad4ab88b..efbf9885d82936 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -550,10 +550,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + if (PyBytes_Check(args[0])) { + if (PyBytes_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 1 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[0])); + goto exit; + } a = PyBytes_AS_STRING(args[0])[0]; } - else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + else if (PyByteArray_Check(args[0])) { + if (PyByteArray_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 1 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[0])); + goto exit; + } a = PyByteArray_AS_STRING(args[0])[0]; } else { @@ -563,10 +577,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } b = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } b = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -576,10 +604,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + if (PyBytes_Check(args[2])) { + if (PyBytes_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 3 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[2])); + goto exit; + } c = PyBytes_AS_STRING(args[2])[0]; } - else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + else if (PyByteArray_Check(args[2])) { + if (PyByteArray_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 3 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[2])); + goto exit; + } c = PyByteArray_AS_STRING(args[2])[0]; } else { @@ -589,10 +631,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + if (PyBytes_Check(args[3])) { + if (PyBytes_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 4 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[3])); + goto exit; + } d = PyBytes_AS_STRING(args[3])[0]; } - else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + else if (PyByteArray_Check(args[3])) { + if (PyByteArray_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 4 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[3])); + goto exit; + } d = PyByteArray_AS_STRING(args[3])[0]; } else { @@ -602,10 +658,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + if (PyBytes_Check(args[4])) { + if (PyBytes_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 5 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[4])); + goto exit; + } e = PyBytes_AS_STRING(args[4])[0]; } - else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + else if (PyByteArray_Check(args[4])) { + if (PyByteArray_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 5 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[4])); + goto exit; + } e = PyByteArray_AS_STRING(args[4])[0]; } else { @@ -615,10 +685,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 6) { goto skip_optional; } - if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + if (PyBytes_Check(args[5])) { + if (PyBytes_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 6 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[5])); + goto exit; + } f = PyBytes_AS_STRING(args[5])[0]; } - else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + else if (PyByteArray_Check(args[5])) { + if (PyByteArray_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 6 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[5])); + goto exit; + } f = PyByteArray_AS_STRING(args[5])[0]; } else { @@ -628,10 +712,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 7) { goto skip_optional; } - if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + if (PyBytes_Check(args[6])) { + if (PyBytes_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 7 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[6])); + goto exit; + } g = PyBytes_AS_STRING(args[6])[0]; } - else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + else if (PyByteArray_Check(args[6])) { + if (PyByteArray_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 7 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[6])); + goto exit; + } g = PyByteArray_AS_STRING(args[6])[0]; } else { @@ -641,10 +739,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 8) { goto skip_optional; } - if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + if (PyBytes_Check(args[7])) { + if (PyBytes_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 8 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[7])); + goto exit; + } h = PyBytes_AS_STRING(args[7])[0]; } - else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + else if (PyByteArray_Check(args[7])) { + if (PyByteArray_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 8 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[7])); + goto exit; + } h = PyByteArray_AS_STRING(args[7])[0]; } else { @@ -654,10 +766,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 9) { goto skip_optional; } - if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + if (PyBytes_Check(args[8])) { + if (PyBytes_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 9 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[8])); + goto exit; + } i = PyBytes_AS_STRING(args[8])[0]; } - else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + else if (PyByteArray_Check(args[8])) { + if (PyByteArray_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 9 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[8])); + goto exit; + } i = PyByteArray_AS_STRING(args[8])[0]; } else { @@ -667,10 +793,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 10) { goto skip_optional; } - if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + if (PyBytes_Check(args[9])) { + if (PyBytes_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 10 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[9])); + goto exit; + } j = PyBytes_AS_STRING(args[9])[0]; } - else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + else if (PyByteArray_Check(args[9])) { + if (PyByteArray_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 10 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[9])); + goto exit; + } j = PyByteArray_AS_STRING(args[9])[0]; } else { @@ -680,10 +820,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 11) { goto skip_optional; } - if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + if (PyBytes_Check(args[10])) { + if (PyBytes_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 11 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[10])); + goto exit; + } k = PyBytes_AS_STRING(args[10])[0]; } - else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + else if (PyByteArray_Check(args[10])) { + if (PyByteArray_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 11 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[10])); + goto exit; + } k = PyByteArray_AS_STRING(args[10])[0]; } else { @@ -693,10 +847,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 12) { goto skip_optional; } - if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + if (PyBytes_Check(args[11])) { + if (PyBytes_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 12 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[11])); + goto exit; + } l = PyBytes_AS_STRING(args[11])[0]; } - else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + else if (PyByteArray_Check(args[11])) { + if (PyByteArray_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 12 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[11])); + goto exit; + } l = PyByteArray_AS_STRING(args[11])[0]; } else { @@ -706,10 +874,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 13) { goto skip_optional; } - if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + if (PyBytes_Check(args[12])) { + if (PyBytes_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 13 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[12])); + goto exit; + } m = PyBytes_AS_STRING(args[12])[0]; } - else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + else if (PyByteArray_Check(args[12])) { + if (PyByteArray_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 13 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[12])); + goto exit; + } m = PyByteArray_AS_STRING(args[12])[0]; } else { @@ -719,10 +901,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 14) { goto skip_optional; } - if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + if (PyBytes_Check(args[13])) { + if (PyBytes_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 14 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[13])); + goto exit; + } n = PyBytes_AS_STRING(args[13])[0]; } - else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + else if (PyByteArray_Check(args[13])) { + if (PyByteArray_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 14 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[13])); + goto exit; + } n = PyByteArray_AS_STRING(args[13])[0]; } else { @@ -740,7 +936,7 @@ static PyObject * test_char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, char f, char g, char h, char i, char j, char k, char l, char m, char n) -/*[clinic end generated code: output=98589f02422fe6b1 input=e42330417a44feac]*/ +/*[clinic end generated code: output=ff11e203248582df input=e42330417a44feac]*/ /*[clinic input] @@ -1028,7 +1224,10 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[2]) != 1) { - _PyArg_BadArgument("test_int_converter", "argument 3", "a unicode character", args[2]); + PyErr_Format(PyExc_TypeError, + "test_int_converter(): argument 3 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[2])); goto exit; } c = PyUnicode_READ_CHAR(args[2], 0); @@ -1048,7 +1247,7 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) static PyObject * test_int_converter_impl(PyObject *module, int a, int b, int c, myenum d) -/*[clinic end generated code: output=5aed87a7589eefb2 input=d20541fc1ca0553e]*/ +/*[clinic end generated code: output=fbcfb7554688663d input=d20541fc1ca0553e]*/ /*[clinic input] diff --git a/Lib/test/test_ctypes/test_functions.py b/Lib/test/test_ctypes/test_functions.py index 63e393f7b7cb6a..3454b83d43e1e7 100644 --- a/Lib/test/test_ctypes/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -2,7 +2,7 @@ import sys import unittest from ctypes import (CDLL, Structure, Array, CFUNCTYPE, - byref, POINTER, pointer, ArgumentError, + byref, POINTER, pointer, ArgumentError, sizeof, c_char, c_wchar, c_byte, c_char_p, c_wchar_p, c_short, c_int, c_long, c_longlong, c_void_p, c_float, c_double, c_longdouble) @@ -72,7 +72,8 @@ def callback(*args): self.assertEqual(str(cm.exception), "argument 1: TypeError: one character bytes, " - "bytearray or integer expected") + "bytearray, or an integer in range(256) expected, " + "not bytes of length 3") def test_wchar_parm(self): f = dll._testfunc_i_bhilfd @@ -84,14 +85,27 @@ def test_wchar_parm(self): with self.assertRaises(ArgumentError) as cm: f(1, 2, 3, 4, 5.0, 6.0) self.assertEqual(str(cm.exception), - "argument 2: TypeError: unicode string expected " - "instead of int instance") + "argument 2: TypeError: a unicode character expected, " + "not instance of int") with self.assertRaises(ArgumentError) as cm: f(1, "abc", 3, 4, 5.0, 6.0) self.assertEqual(str(cm.exception), - "argument 2: TypeError: one character unicode string " - "expected") + "argument 2: TypeError: a unicode character expected, " + "not a string of length 3") + + with self.assertRaises(ArgumentError) as cm: + f(1, "", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: a unicode character expected, " + "not a string of length 0") + + if sizeof(c_wchar) < 4: + with self.assertRaises(ArgumentError) as cm: + f(1, "\U0001f40d", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: the string '\\U0001f40d' " + "cannot be converted to a single wchar_t character") def test_c_char_p_parm(self): """Test the error message when converting an incompatible type to c_char_p.""" diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index effb8db418f790..f89521cf8b3a67 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -3,7 +3,7 @@ from ctypes import (CDLL, PyDLL, ArgumentError, Structure, Array, Union, _Pointer, _SimpleCData, _CFuncPtr, - POINTER, pointer, byref, + POINTER, pointer, byref, sizeof, c_void_p, c_char_p, c_wchar_p, py_object, c_bool, c_char, c_wchar, @@ -87,19 +87,33 @@ def test_c_char(self): with self.assertRaises(TypeError) as cm: c_char.from_param(b"abc") self.assertEqual(str(cm.exception), - "one character bytes, bytearray or integer expected") + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytes of length 3") def test_c_wchar(self): with self.assertRaises(TypeError) as cm: c_wchar.from_param("abc") self.assertEqual(str(cm.exception), - "one character unicode string expected") + "a unicode character expected, not a string of length 3") + with self.assertRaises(TypeError) as cm: + c_wchar.from_param("") + self.assertEqual(str(cm.exception), + "a unicode character expected, not a string of length 0") with self.assertRaises(TypeError) as cm: c_wchar.from_param(123) self.assertEqual(str(cm.exception), - "unicode string expected instead of int instance") + "a unicode character expected, not instance of int") + + if sizeof(c_wchar) < 4: + with self.assertRaises(TypeError) as cm: + c_wchar.from_param('\U0001f40d') + self.assertEqual(str(cm.exception), + "the string '\\U0001f40d' cannot be converted to " + "a single wchar_t character") + + def test_int_pointers(self): LPINT = POINTER(c_int) diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index ea37eb5d96457d..7bdd2881904548 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -1578,7 +1578,7 @@ def __int__(self): self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) - self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) + self.assertRaisesRegex(TypeError, r'%c requires an int or a unicode character, not .*\.PseudoFloat', operator.mod, '%c', pi) class RaisingNumber: def __int__(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-08-13-49-02.gh-issue-117558.9lSEpR.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-08-13-49-02.gh-issue-117558.9lSEpR.rst new file mode 100644 index 00000000000000..222c516d2ef1b9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-08-13-49-02.gh-issue-117558.9lSEpR.rst @@ -0,0 +1,2 @@ +Improve error messages when a string, bytes or bytearray object of length 1 +is expected. diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 7472a4c36868a8..7c98b0f7e31a46 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -1100,25 +1100,45 @@ O_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * c_set(void *ptr, PyObject *value, Py_ssize_t size) { - if (PyBytes_Check(value) && PyBytes_GET_SIZE(value) == 1) { + if (PyBytes_Check(value)) { + if (PyBytes_GET_SIZE(value) != 1) { + PyErr_Format(PyExc_TypeError, + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytes of length %zd", + PyBytes_GET_SIZE(value)); + return NULL; + } *(char *)ptr = PyBytes_AS_STRING(value)[0]; _RET(value); } - if (PyByteArray_Check(value) && PyByteArray_GET_SIZE(value) == 1) { + if (PyByteArray_Check(value)) { + if (PyByteArray_GET_SIZE(value) != 1) { + PyErr_Format(PyExc_TypeError, + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytearray of length %zd", + PyByteArray_GET_SIZE(value)); + return NULL; + } *(char *)ptr = PyByteArray_AS_STRING(value)[0]; _RET(value); } - if (PyLong_Check(value)) - { - long longval = PyLong_AsLong(value); - if (longval < 0 || longval >= 256) - goto error; + if (PyLong_Check(value)) { + int overflow; + long longval = PyLong_AsLongAndOverflow(value, &overflow); + if (longval == -1 && PyErr_Occurred()) { + return NULL; + } + if (overflow || longval < 0 || longval >= 256) { + PyErr_SetString(PyExc_TypeError, "integer not in range(256)"); + return NULL; + } *(char *)ptr = (char)longval; _RET(value); } - error: PyErr_Format(PyExc_TypeError, - "one character bytes, bytearray or integer expected"); + "one character bytes, bytearray, or an integer " + "in range(256) expected, not %T", + value); return NULL; } @@ -1137,22 +1157,27 @@ u_set(void *ptr, PyObject *value, Py_ssize_t size) wchar_t chars[2]; if (!PyUnicode_Check(value)) { PyErr_Format(PyExc_TypeError, - "unicode string expected instead of %s instance", - Py_TYPE(value)->tp_name); + "a unicode character expected, not instance of %T", + value); return NULL; - } else - Py_INCREF(value); + } len = PyUnicode_AsWideChar(value, chars, 2); if (len != 1) { - Py_DECREF(value); - PyErr_SetString(PyExc_TypeError, - "one character unicode string expected"); + if (PyUnicode_GET_LENGTH(value) != 1) { + PyErr_Format(PyExc_TypeError, + "a unicode character expected, not a string of length %zd", + PyUnicode_GET_LENGTH(value)); + } + else { + PyErr_Format(PyExc_TypeError, + "the string %A cannot be converted to a single wchar_t character", + value); + } return NULL; } *(wchar_t *)ptr = chars[0]; - Py_DECREF(value); _RET(value); } diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index ee3d4c6eae7546..3a011963968b1a 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -233,13 +233,20 @@ static int PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) { long value; - if(PyBytes_Check(obj) && PyBytes_Size(obj) == 1) { + if (PyBytes_Check(obj)) { + if (PyBytes_GET_SIZE(obj) != 1) { + PyErr_Format(PyExc_TypeError, + "expect int or bytes or str of length 1, " + "got a bytes of length %zd", + PyBytes_GET_SIZE(obj)); + return 0; + } value = (unsigned char)PyBytes_AsString(obj)[0]; } else if (PyUnicode_Check(obj)) { - if (PyUnicode_GetLength(obj) != 1) { + if (PyUnicode_GET_LENGTH(obj) != 1) { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, " + "expect int or bytes or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -272,7 +279,7 @@ PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) } else { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, got %s", + "expect int or bytes or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -315,7 +322,7 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, #ifdef HAVE_NCURSESW if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, " + "expect int or bytes or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -326,7 +333,14 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, return PyCurses_ConvertToChtype(win, obj, ch); #endif } - else if(PyBytes_Check(obj) && PyBytes_Size(obj) == 1) { + else if (PyBytes_Check(obj)) { + if (PyBytes_GET_SIZE(obj) != 1) { + PyErr_Format(PyExc_TypeError, + "expect int or bytes or str of length 1, " + "got a bytes of length %zd", + PyBytes_GET_SIZE(obj)); + return 0; + } value = (unsigned char)PyBytes_AsString(obj)[0]; } else if (PyLong_CheckExact(obj)) { @@ -340,7 +354,7 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, } else { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, got %s", + "expect int or bytes or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -4443,7 +4457,7 @@ PyCurses_ConvertToWchar_t(PyObject *obj, wchar_t buffer[2]; if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) { PyErr_Format(PyExc_TypeError, - "expect str of length 1 or int, " + "expect int or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -4470,7 +4484,7 @@ PyCurses_ConvertToWchar_t(PyObject *obj, } else { PyErr_Format(PyExc_TypeError, - "expect str of length 1 or int, got %s", + "expect int or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index a3b833d47cd9ea..e6c84d588be98b 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -260,20 +260,32 @@ u_getitem(arrayobject *ap, Py_ssize_t i) static int u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - PyObject *u; - if (!PyArg_Parse(v, "U;array item must be unicode character", &u)) { + if (!PyUnicode_Check(v)) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, not %T", + v); return -1; } - Py_ssize_t len = PyUnicode_AsWideChar(u, NULL, 0); + Py_ssize_t len = PyUnicode_AsWideChar(v, NULL, 0); if (len != 2) { - PyErr_SetString(PyExc_TypeError, - "array item must be unicode character"); + if (PyUnicode_GET_LENGTH(v) != 1) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); + } + else { + PyErr_Format(PyExc_TypeError, + "string %A cannot be converted to " + "a single wchar_t character", + v); + } return -1; } wchar_t w; - len = PyUnicode_AsWideChar(u, &w, 1); + len = PyUnicode_AsWideChar(v, &w, 1); assert(len == 1); if (i >= 0) { @@ -291,19 +303,23 @@ w_getitem(arrayobject *ap, Py_ssize_t i) static int w_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - PyObject *u; - if (!PyArg_Parse(v, "U;array item must be unicode character", &u)) { + if (!PyUnicode_Check(v)) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, not %T", + v); return -1; } - if (PyUnicode_GetLength(u) != 1) { - PyErr_SetString(PyExc_TypeError, - "array item must be unicode character"); + if (PyUnicode_GET_LENGTH(v) != 1) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); return -1; } if (i >= 0) { - ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(u, 0); + ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(v, 0); } return 0; } diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 6a59baa2137b75..16e7c808d39e7c 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -234,10 +234,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + if (PyBytes_Check(args[0])) { + if (PyBytes_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 1 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[0])); + goto exit; + } a = PyBytes_AS_STRING(args[0])[0]; } - else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + else if (PyByteArray_Check(args[0])) { + if (PyByteArray_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 1 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[0])); + goto exit; + } a = PyByteArray_AS_STRING(args[0])[0]; } else { @@ -247,10 +261,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } b = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } b = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -260,10 +288,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + if (PyBytes_Check(args[2])) { + if (PyBytes_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 3 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[2])); + goto exit; + } c = PyBytes_AS_STRING(args[2])[0]; } - else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + else if (PyByteArray_Check(args[2])) { + if (PyByteArray_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 3 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[2])); + goto exit; + } c = PyByteArray_AS_STRING(args[2])[0]; } else { @@ -273,10 +315,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + if (PyBytes_Check(args[3])) { + if (PyBytes_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 4 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[3])); + goto exit; + } d = PyBytes_AS_STRING(args[3])[0]; } - else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + else if (PyByteArray_Check(args[3])) { + if (PyByteArray_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 4 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[3])); + goto exit; + } d = PyByteArray_AS_STRING(args[3])[0]; } else { @@ -286,10 +342,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + if (PyBytes_Check(args[4])) { + if (PyBytes_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 5 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[4])); + goto exit; + } e = PyBytes_AS_STRING(args[4])[0]; } - else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + else if (PyByteArray_Check(args[4])) { + if (PyByteArray_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 5 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[4])); + goto exit; + } e = PyByteArray_AS_STRING(args[4])[0]; } else { @@ -299,10 +369,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 6) { goto skip_optional; } - if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + if (PyBytes_Check(args[5])) { + if (PyBytes_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 6 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[5])); + goto exit; + } f = PyBytes_AS_STRING(args[5])[0]; } - else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + else if (PyByteArray_Check(args[5])) { + if (PyByteArray_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 6 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[5])); + goto exit; + } f = PyByteArray_AS_STRING(args[5])[0]; } else { @@ -312,10 +396,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 7) { goto skip_optional; } - if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + if (PyBytes_Check(args[6])) { + if (PyBytes_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 7 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[6])); + goto exit; + } g = PyBytes_AS_STRING(args[6])[0]; } - else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + else if (PyByteArray_Check(args[6])) { + if (PyByteArray_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 7 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[6])); + goto exit; + } g = PyByteArray_AS_STRING(args[6])[0]; } else { @@ -325,10 +423,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 8) { goto skip_optional; } - if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + if (PyBytes_Check(args[7])) { + if (PyBytes_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 8 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[7])); + goto exit; + } h = PyBytes_AS_STRING(args[7])[0]; } - else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + else if (PyByteArray_Check(args[7])) { + if (PyByteArray_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 8 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[7])); + goto exit; + } h = PyByteArray_AS_STRING(args[7])[0]; } else { @@ -338,10 +450,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 9) { goto skip_optional; } - if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + if (PyBytes_Check(args[8])) { + if (PyBytes_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 9 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[8])); + goto exit; + } i = PyBytes_AS_STRING(args[8])[0]; } - else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + else if (PyByteArray_Check(args[8])) { + if (PyByteArray_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 9 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[8])); + goto exit; + } i = PyByteArray_AS_STRING(args[8])[0]; } else { @@ -351,10 +477,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 10) { goto skip_optional; } - if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + if (PyBytes_Check(args[9])) { + if (PyBytes_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 10 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[9])); + goto exit; + } j = PyBytes_AS_STRING(args[9])[0]; } - else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + else if (PyByteArray_Check(args[9])) { + if (PyByteArray_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 10 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[9])); + goto exit; + } j = PyByteArray_AS_STRING(args[9])[0]; } else { @@ -364,10 +504,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 11) { goto skip_optional; } - if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + if (PyBytes_Check(args[10])) { + if (PyBytes_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 11 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[10])); + goto exit; + } k = PyBytes_AS_STRING(args[10])[0]; } - else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + else if (PyByteArray_Check(args[10])) { + if (PyByteArray_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 11 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[10])); + goto exit; + } k = PyByteArray_AS_STRING(args[10])[0]; } else { @@ -377,10 +531,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 12) { goto skip_optional; } - if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + if (PyBytes_Check(args[11])) { + if (PyBytes_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 12 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[11])); + goto exit; + } l = PyBytes_AS_STRING(args[11])[0]; } - else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + else if (PyByteArray_Check(args[11])) { + if (PyByteArray_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 12 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[11])); + goto exit; + } l = PyByteArray_AS_STRING(args[11])[0]; } else { @@ -390,10 +558,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 13) { goto skip_optional; } - if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + if (PyBytes_Check(args[12])) { + if (PyBytes_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 13 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[12])); + goto exit; + } m = PyBytes_AS_STRING(args[12])[0]; } - else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + else if (PyByteArray_Check(args[12])) { + if (PyByteArray_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 13 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[12])); + goto exit; + } m = PyByteArray_AS_STRING(args[12])[0]; } else { @@ -403,10 +585,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 14) { goto skip_optional; } - if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + if (PyBytes_Check(args[13])) { + if (PyBytes_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 14 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[13])); + goto exit; + } n = PyBytes_AS_STRING(args[13])[0]; } - else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + else if (PyByteArray_Check(args[13])) { + if (PyByteArray_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 14 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[13])); + goto exit; + } n = PyByteArray_AS_STRING(args[13])[0]; } else { @@ -648,7 +844,10 @@ int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[2]) != 1) { - _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + PyErr_Format(PyExc_TypeError, + "int_converter(): argument 3 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[2])); goto exit; } c = PyUnicode_READ_CHAR(args[2], 0); @@ -3219,4 +3418,4 @@ _testclinic_TestClass_get_defining_class_arg(PyObject *self, PyTypeObject *cls, exit: return return_value; } -/*[clinic end generated code: output=aa352c3a67300056 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=545d409a47f1826d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 60a03fe012550e..8427f92de0d2e3 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -596,7 +596,10 @@ array__array_reconstructor(PyObject *module, PyObject *const *args, Py_ssize_t n goto exit; } if (PyUnicode_GET_LENGTH(args[1]) != 1) { - _PyArg_BadArgument("_array_reconstructor", "argument 2", "a unicode character", args[1]); + PyErr_Format(PyExc_TypeError, + "_array_reconstructor(): argument 2 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[1])); goto exit; } typecode = PyUnicode_READ_CHAR(args[1], 0); @@ -685,4 +688,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=52c55d9b1d026c1c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9a3276ffd499c796 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/unicodedata.c.h b/Modules/clinic/unicodedata.c.h index 739f498f1d2672..345440eeee89a6 100644 --- a/Modules/clinic/unicodedata.c.h +++ b/Modules/clinic/unicodedata.c.h @@ -36,7 +36,10 @@ unicodedata_UCD_decimal(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("decimal", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "decimal(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -82,7 +85,10 @@ unicodedata_UCD_digit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("digit", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "digit(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -129,7 +135,10 @@ unicodedata_UCD_numeric(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("numeric", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "numeric(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -167,7 +176,10 @@ unicodedata_UCD_category(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("category", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "category(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -202,7 +214,10 @@ unicodedata_UCD_bidirectional(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("bidirectional", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "bidirectional(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -238,7 +253,10 @@ unicodedata_UCD_combining(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("combining", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "combining(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -279,7 +297,10 @@ unicodedata_UCD_mirrored(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("mirrored", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "mirrored(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -316,7 +337,10 @@ unicodedata_UCD_east_asian_width(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("east_asian_width", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "east_asian_width(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -351,7 +375,10 @@ unicodedata_UCD_decomposition(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("decomposition", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "decomposition(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -473,7 +500,10 @@ unicodedata_UCD_name(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("name", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "name(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -519,4 +549,4 @@ unicodedata_UCD_lookup(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=ea30f89007b2bfff input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8a59d430cee41058 input=a9049054013a1b77]*/ diff --git a/Modules/termios.c b/Modules/termios.c index f2c5a4bafa7012..0633d8f82cc7e4 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -260,7 +260,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) } else { PyErr_SetString(PyExc_TypeError, - "tcsetattr: elements of attributes must be characters or integers"); + "tcsetattr: elements of attributes must be bytes objects of length 1 or integers"); return NULL; } } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 0f6435d84c113e..459df6ceacf3a8 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -477,21 +477,32 @@ formatlong(PyObject *v, int flags, int prec, int type) static int byte_converter(PyObject *arg, char *p) { - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or " + "a single byte, not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + return 0; + } *p = PyBytes_AS_STRING(arg)[0]; return 1; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or " + "a single byte, not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + return 0; + } *p = PyByteArray_AS_STRING(arg)[0]; return 1; } - else { + else if (PyIndex_Check(arg)) { int overflow; long ival = PyLong_AsLongAndOverflow(arg, &overflow); if (ival == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - goto onError; - } return 0; } if (!(0 <= ival && ival <= 255)) { @@ -503,9 +514,9 @@ byte_converter(PyObject *arg, char *p) *p = (char)ival; return 1; } - onError: - PyErr_SetString(PyExc_TypeError, - "%c requires an integer in range(256) or a single byte"); + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or a single byte, not %T", + arg); return 0; } diff --git a/Objects/stringlib/clinic/transmogrify.h.h b/Objects/stringlib/clinic/transmogrify.h.h index 3a985ab5c7a9f5..cef7a9496fa874 100644 --- a/Objects/stringlib/clinic/transmogrify.h.h +++ b/Objects/stringlib/clinic/transmogrify.h.h @@ -113,10 +113,24 @@ stringlib_ljust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "ljust(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "ljust(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -169,10 +183,24 @@ stringlib_rjust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "rjust(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "rjust(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -225,10 +253,24 @@ stringlib_center(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "center(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "center(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -279,4 +321,4 @@ stringlib_zfill(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=b409bdf9ab68d5a6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06dd79019356b6bb input=a9049054013a1b77]*/ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 048f9a814c30af..92db31f1e498f9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14057,14 +14057,21 @@ formatchar(PyObject *v) if (PyUnicode_GET_LENGTH(v) == 1) { return PyUnicode_READ_CHAR(v, 0); } - goto onError; + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); + return (Py_UCS4) -1; } else { int overflow; long x = PyLong_AsLongAndOverflow(v, &overflow); if (x == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - goto onError; + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, not %T", + v); + return (Py_UCS4) -1; } return (Py_UCS4) -1; } @@ -14078,11 +14085,6 @@ formatchar(PyObject *v) return (Py_UCS4) x; } - - onError: - PyErr_SetString(PyExc_TypeError, - "%c requires int or char"); - return (Py_UCS4) -1; } /* Parse options of an argument: flags, width, precision. diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index e3f7ea43f38211..a77d0855af293f 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -355,10 +355,24 @@ msvcrt_putch(PyObject *module, PyObject *arg) PyObject *return_value = NULL; char char_value; - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "putch(): argument must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + goto exit; + } char_value = PyBytes_AS_STRING(arg)[0]; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "putch(): argument must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + goto exit; + } char_value = PyByteArray_AS_STRING(arg)[0]; } else { @@ -396,7 +410,10 @@ msvcrt_putwch(PyObject *module, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("putwch", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "putwch(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } unicode_char = PyUnicode_READ_CHAR(arg, 0); @@ -430,10 +447,24 @@ msvcrt_ungetch(PyObject *module, PyObject *arg) PyObject *return_value = NULL; char char_value; - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "ungetch(): argument must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + goto exit; + } char_value = PyBytes_AS_STRING(arg)[0]; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "ungetch(): argument must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + goto exit; + } char_value = PyByteArray_AS_STRING(arg)[0]; } else { @@ -471,7 +502,10 @@ msvcrt_ungetwch(PyObject *module, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("ungetwch", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "ungetwch(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } unicode_char = PyUnicode_READ_CHAR(arg, 0); @@ -697,4 +731,4 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_GETERRORMODE_METHODDEF #define MSVCRT_GETERRORMODE_METHODDEF #endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ -/*[clinic end generated code: output=de9687b46212c2ed input=a9049054013a1b77]*/ +/*[clinic end generated code: output=692c6f52bb9193ce input=a9049054013a1b77]*/ diff --git a/Python/getargs.c b/Python/getargs.c index f9a836679fe55e..88f4c58ed2caa6 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -589,6 +589,17 @@ converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) return msgbuf; } +static const char * +convertcharerr(const char *expected, const char *what, Py_ssize_t size, + char *msgbuf, size_t bufsize) +{ + assert(expected != NULL); + PyOS_snprintf(msgbuf, bufsize, + "must be %.50s, not %.50s of length %zd", + expected, what, size); + return msgbuf; +} + #define CONV_UNICODE "(unicode conversion error)" /* Convert a non-tuple argument. Return NULL if conversion went OK, @@ -795,10 +806,22 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'c': {/* char */ char *p = va_arg(*p_va, char *); - if (PyBytes_Check(arg) && PyBytes_Size(arg) == 1) + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + return convertcharerr("a byte string of length 1", + "a bytes object", PyBytes_GET_SIZE(arg), + msgbuf, bufsize); + } *p = PyBytes_AS_STRING(arg)[0]; - else if (PyByteArray_Check(arg) && PyByteArray_Size(arg) == 1) + } + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + return convertcharerr("a byte string of length 1", + "a bytearray object", PyByteArray_GET_SIZE(arg), + msgbuf, bufsize); + } *p = PyByteArray_AS_STRING(arg)[0]; + } else return converterr("a byte string of length 1", arg, msgbuf, bufsize); break; @@ -812,8 +835,11 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (!PyUnicode_Check(arg)) return converterr("a unicode character", arg, msgbuf, bufsize); - if (PyUnicode_GET_LENGTH(arg) != 1) - return converterr("a unicode character", arg, msgbuf, bufsize); + if (PyUnicode_GET_LENGTH(arg) != 1) { + return convertcharerr("a unicode character", + "a string", PyUnicode_GET_LENGTH(arg), + msgbuf, bufsize); + } kind = PyUnicode_KIND(arg); data = PyUnicode_DATA(arg); diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index 0778961f5b5875..bd5c2a2b73b94a 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -89,10 +89,24 @@ def converter_init(self) -> None: def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'c': return self.format_code(""" - if (PyBytes_Check({argname}) && PyBytes_GET_SIZE({argname}) == 1) {{{{ + if (PyBytes_Check({argname})) {{{{ + if (PyBytes_GET_SIZE({argname}) != 1) {{{{ + PyErr_Format(PyExc_TypeError, + "{{name}}(): {displayname} must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE({argname})); + goto exit; + }}}} {paramname} = PyBytes_AS_STRING({argname})[0]; }}}} - else if (PyByteArray_Check({argname}) && PyByteArray_GET_SIZE({argname}) == 1) {{{{ + else if (PyByteArray_Check({argname})) {{{{ + if (PyByteArray_GET_SIZE({argname}) != 1) {{{{ + PyErr_Format(PyExc_TypeError, + "{{name}}(): {displayname} must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE({argname})); + goto exit; + }}}} {paramname} = PyByteArray_AS_STRING({argname})[0]; }}}} else {{{{ @@ -101,6 +115,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st }}}} """, argname=argname, + displayname=displayname, bad_argument=self.bad_argument(displayname, 'a byte string of length 1', limited_capi=limited_capi), ) return super().parse_arg(argname, displayname, limited_capi=limited_capi) @@ -272,12 +287,16 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st goto exit; }}}} if (PyUnicode_GET_LENGTH({argname}) != 1) {{{{ - {bad_argument} + PyErr_Format(PyExc_TypeError, + "{{name}}(): {displayname} must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH({argname})); goto exit; }}}} {paramname} = PyUnicode_READ_CHAR({argname}, 0); """, argname=argname, + displayname=displayname, bad_argument=self.bad_argument(displayname, 'a unicode character', limited_capi=limited_capi), ) return super().parse_arg(argname, displayname, limited_capi=limited_capi) From 669175bf8edc2c02d48401bac0e4c7d99a33f15b Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 28 May 2024 12:16:52 +0300 Subject: [PATCH 235/903] gh-116860: Remove outdated `test_parserhack` from `test_future` (#116861) --- Lib/test/test_future_stmt/test_future.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 69ae58b0fbcae3..bb31d0a0023fad 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -171,26 +171,6 @@ def test_ensure_flags_dont_clash(self): } self.assertCountEqual(set(flags.values()), flags.values()) - def test_parserhack(self): - # test that the parser.c::future_hack function works as expected - # Note: although this test must pass, it's not testing the original - # bug as of 2.6 since the with statement is not optional and - # the parser hack disabled. If a new keyword is introduced in - # 2.6, change this to refer to the new future import. - try: - exec("from __future__ import print_function; print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - - try: - exec("from __future__ import (print_function); print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - def test_unicode_literals_exec(self): scope = {} exec("from __future__ import unicode_literals; x = ''", {}, scope) From b407ad38fb93585332c370b8fa56905fb238cdfd Mon Sep 17 00:00:00 2001 From: Justin Kunimune Date: Tue, 28 May 2024 06:31:20 -0400 Subject: [PATCH 236/903] [doc] Clarify the nature of the root logger in the `logging` documentation (GH-119440) Co-authored-by: Vinay Sajip --- Doc/library/logging.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 564b34bcf1bb37..4ba520c139ebce 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -109,11 +109,11 @@ The ``name`` is potentially a period-separated hierarchical value, like Loggers that are further down in the hierarchical list are children of loggers higher up in the list. For example, given a logger with a name of ``foo``, loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all -descendants of ``foo``. The logger name hierarchy is analogous to the Python -package hierarchy, and identical to it if you organise your loggers on a -per-module basis using the recommended construction -``logging.getLogger(__name__)``. That's because in a module, ``__name__`` -is the module's name in the Python package namespace. +descendants of ``foo``. In addition, all loggers are descendants of the root +logger. The logger name hierarchy is analogous to the Python package hierarchy, +and identical to it if you organise your loggers on a per-module basis using +the recommended construction ``logging.getLogger(__name__)``. That's because +in a module, ``__name__`` is the module's name in the Python package namespace. .. class:: Logger @@ -1157,10 +1157,12 @@ functions. .. function:: getLogger(name=None) - Return a logger with the specified name or, if name is ``None``, return a - logger which is the root logger of the hierarchy. If specified, the name is - typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. - Choice of these names is entirely up to the developer who is using logging. + Return a logger with the specified name or, if name is ``None``, return the + root logger of the hierarchy. If specified, the name is typically a + dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. Choice + of these names is entirely up to the developer who is using logging, though + it is recommended that ``__name__`` be used unless you have a specific + reason for not doing that, as mentioned in :ref:`logger`. All calls to this function with a given name return the same logger instance. This means that logger instances never need to be passed between different parts From f912e5a2f6d128b17f85229b722422e4a2478e23 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 28 May 2024 16:42:35 +0300 Subject: [PATCH 237/903] gh-118824: Remove deprecated `master_open` and `slave_open` from `pty` (#118826) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.14.rst | 8 ++++ Lib/pty.py | 47 ++++--------------- ...-05-09-11-50-26.gh-issue-118824.-jBJQC.rst | 3 ++ 3 files changed, 20 insertions(+), 38 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-09-11-50-26.gh-issue-118824.-jBJQC.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index bc12d4b3b590dd..15216479cc6e5c 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -175,6 +175,14 @@ pathlib :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such arguments are joined onto *other*. +pty +___ + +* Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. + They had previously raised a :exc:`DeprecationWarning` since Python 3.12. + Use :func:`pty.openpty` instead. + (Contributed by Nikita Sobolev in :gh:`118824`.) + sqlite3 ------- diff --git a/Lib/pty.py b/Lib/pty.py index 1d97994abef3c8..eb3d5f1ff657bb 100644 --- a/Lib/pty.py +++ b/Lib/pty.py @@ -32,27 +32,18 @@ def openpty(): except (AttributeError, OSError): pass master_fd, slave_name = _open_terminal() - slave_fd = slave_open(slave_name) - return master_fd, slave_fd - -def master_open(): - """master_open() -> (master_fd, slave_name) - Open a pty master and return the fd, and the filename of the slave end. - Deprecated, use openpty() instead.""" - - import warnings - warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 + slave_fd = os.open(slave_name, os.O_RDWR) try: - master_fd, slave_fd = os.openpty() - except (AttributeError, OSError): + from fcntl import ioctl, I_PUSH + except ImportError: + return master_fd, slave_fd + try: + ioctl(result, I_PUSH, "ptem") + ioctl(result, I_PUSH, "ldterm") + except OSError: pass - else: - slave_name = os.ttyname(slave_fd) - os.close(slave_fd) - return master_fd, slave_name - - return _open_terminal() + return master_fd, slave_fd def _open_terminal(): """Open pty master and return (master_fd, tty_name).""" @@ -66,26 +57,6 @@ def _open_terminal(): return (fd, '/dev/tty' + x + y) raise OSError('out of pty devices') -def slave_open(tty_name): - """slave_open(tty_name) -> slave_fd - Open the pty slave and acquire the controlling terminal, returning - opened filedescriptor. - Deprecated, use openpty() instead.""" - - import warnings - warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 - - result = os.open(tty_name, os.O_RDWR) - try: - from fcntl import ioctl, I_PUSH - except ImportError: - return result - try: - ioctl(result, I_PUSH, "ptem") - ioctl(result, I_PUSH, "ldterm") - except OSError: - pass - return result def fork(): """fork() -> (pid, master_fd) diff --git a/Misc/NEWS.d/next/Library/2024-05-09-11-50-26.gh-issue-118824.-jBJQC.rst b/Misc/NEWS.d/next/Library/2024-05-09-11-50-26.gh-issue-118824.-jBJQC.rst new file mode 100644 index 00000000000000..c9254f1b9dbea8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-11-50-26.gh-issue-118824.-jBJQC.rst @@ -0,0 +1,3 @@ +Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. +Use :func:`pty.openpty` instead. +Patch by Nikita Sobolev. From 2da0dc094fa855ed4df251aab58b6f8a2b6969a1 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 28 May 2024 18:50:50 +0300 Subject: [PATCH 238/903] gh-119659: Move `@no_rerun` to `test.support` (#119660) --- Lib/test/datetimetester.py | 22 +--------------------- Lib/test/support/__init__.py | 19 +++++++++++++++++++ Lib/test/test_import/__init__.py | 21 +-------------------- 3 files changed, 21 insertions(+), 41 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index ba7f185a092629..535b17d0727611 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -22,7 +22,7 @@ from test import support from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST -from test.support import warnings_helper +from test.support import warnings_helper, no_rerun import datetime as datetime_module from datetime import MINYEAR, MAXYEAR @@ -47,26 +47,6 @@ pass # -# This is copied from test_import/__init__.py. -# XXX Move it to support/__init__.py. -def no_rerun(reason): - """Skip rerunning for a particular test. - - WARNING: Use this decorator with care; skipping rerunning makes it - impossible to find reference leaks. Provide a clear reason for skipping the - test using the 'reason' parameter. - """ - def deco(func): - _has_run = False - def wrapper(self): - nonlocal _has_run - if _has_run: - self.skipTest(reason) - func(self) - _has_run = True - return wrapper - return deco - pickle_loads = {pickle.loads, pickle._loads} pickle_choices = [(pickle, pickle, proto) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index e2a854663ddb7d..5825efadffcb29 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1189,6 +1189,25 @@ def coverage_wrapper(*args, **kwargs): return coverage_wrapper +def no_rerun(reason): + """Skip rerunning for a particular test. + + WARNING: Use this decorator with care; skipping rerunning makes it + impossible to find reference leaks. Provide a clear reason for skipping the + test using the 'reason' parameter. + """ + def deco(func): + _has_run = False + def wrapper(self): + nonlocal _has_run + if _has_run: + self.skipTest(reason) + func(self) + _has_run = True + return wrapper + return deco + + def refcount_test(test): """Decorator for tests which involve reference counting. diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index b09065f812c0e4..97e262cfdb9023 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -31,7 +31,7 @@ from test.support import ( STDLIB_DIR, swap_attr, swap_item, cpython_only, is_apple_mobile, is_emscripten, is_wasi, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS, - requires_gil_enabled, Py_GIL_DISABLED) + requires_gil_enabled, Py_GIL_DISABLED, no_rerun) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, ready_to_import, DirsOnSysPath, CleanImport, import_module) @@ -120,25 +120,6 @@ def remove_files(name): rmtree('__pycache__') -def no_rerun(reason): - """Skip rerunning for a particular test. - - WARNING: Use this decorator with care; skipping rerunning makes it - impossible to find reference leaks. Provide a clear reason for skipping the - test using the 'reason' parameter. - """ - def deco(func): - _has_run = False - def wrapper(self): - nonlocal _has_run - if _has_run: - self.skipTest(reason) - func(self) - _has_run = True - return wrapper - return deco - - if _testsinglephase is not None: def restore__testsinglephase(*, _orig=_testsinglephase): # We started with the module imported and want to restore From 0518edc17049a2f474b049b7d7fe3ef4339ceb83 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 28 May 2024 18:05:20 +0200 Subject: [PATCH 239/903] gh-119396: Optimize unicode_repr() (#119617) Use stringlib to specialize unicode_repr() for each string kind (UCS1, UCS2, UCS4). Benchmark: +-------------------------------------+---------+----------------------+ | Benchmark | ref | change2 | +=====================================+=========+======================+ | repr('abc') | 100 ns | 103 ns: 1.02x slower | +-------------------------------------+---------+----------------------+ | repr('a' * 100) | 369 ns | 369 ns: 1.00x slower | +-------------------------------------+---------+----------------------+ | repr(('a' + squote) * 100) | 1.21 us | 946 ns: 1.27x faster | +-------------------------------------+---------+----------------------+ | repr(('a' + nl) * 100) | 1.23 us | 907 ns: 1.36x faster | +-------------------------------------+---------+----------------------+ | repr(dquote + ('a' + squote) * 100) | 1.08 us | 858 ns: 1.25x faster | +-------------------------------------+---------+----------------------+ | Geometric mean | (ref) | 1.16x faster | +-------------------------------------+---------+----------------------+ --- Makefile.pre.in | 1 + Objects/stringlib/repr.h | 95 +++++++++++++++++++ Objects/unicodeobject.c | 136 +++++++--------------------- Tools/c-analyzer/cpython/_parser.py | 1 + 4 files changed, 131 insertions(+), 102 deletions(-) create mode 100644 Objects/stringlib/repr.h diff --git a/Makefile.pre.in b/Makefile.pre.in index 9e99c95e2af042..a80d9334ba5134 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1841,6 +1841,7 @@ UNICODE_DEPS = \ $(srcdir)/Objects/stringlib/localeutil.h \ $(srcdir)/Objects/stringlib/partition.h \ $(srcdir)/Objects/stringlib/replace.h \ + $(srcdir)/Objects/stringlib/repr.h \ $(srcdir)/Objects/stringlib/split.h \ $(srcdir)/Objects/stringlib/ucs1lib.h \ $(srcdir)/Objects/stringlib/ucs2lib.h \ diff --git a/Objects/stringlib/repr.h b/Objects/stringlib/repr.h new file mode 100644 index 00000000000000..87b1a8ba629dc6 --- /dev/null +++ b/Objects/stringlib/repr.h @@ -0,0 +1,95 @@ +/* stringlib: repr() implementation */ + +#ifndef STRINGLIB_FASTSEARCH_H +#error must include "stringlib/fastsearch.h" before including this module +#endif + + +static void +STRINGLIB(repr)(PyObject *unicode, Py_UCS4 quote, + STRINGLIB_CHAR *odata) +{ + Py_ssize_t isize = PyUnicode_GET_LENGTH(unicode); + const void *idata = PyUnicode_DATA(unicode); + int ikind = PyUnicode_KIND(unicode); + + *odata++ = quote; + for (Py_ssize_t i = 0; i < isize; i++) { + Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); + + /* Escape quotes and backslashes */ + if ((ch == quote) || (ch == '\\')) { + *odata++ = '\\'; + *odata++ = ch; + continue; + } + + /* Map special whitespace to '\t', \n', '\r' */ + if (ch == '\t') { + *odata++ = '\\'; + *odata++ = 't'; + } + else if (ch == '\n') { + *odata++ = '\\'; + *odata++ = 'n'; + } + else if (ch == '\r') { + *odata++ = '\\'; + *odata++ = 'r'; + } + + /* Map non-printable US ASCII to '\xhh' */ + else if (ch < ' ' || ch == 0x7F) { + *odata++ = '\\'; + *odata++ = 'x'; + *odata++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *odata++ = Py_hexdigits[ch & 0x000F]; + } + + /* Copy ASCII characters as-is */ + else if (ch < 0x7F) { + *odata++ = ch; + } + + /* Non-ASCII characters */ + else { + /* Map Unicode whitespace and control characters + (categories Z* and C* except ASCII space) + */ + if (!Py_UNICODE_ISPRINTABLE(ch)) { + *odata++ = '\\'; + /* Map 8-bit characters to '\xhh' */ + if (ch <= 0xff) { + *odata++ = 'x'; + *odata++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *odata++ = Py_hexdigits[ch & 0x000F]; + } + /* Map 16-bit characters to '\uxxxx' */ + else if (ch <= 0xffff) { + *odata++ = 'u'; + *odata++ = Py_hexdigits[(ch >> 12) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 8) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 4) & 0xF]; + *odata++ = Py_hexdigits[ch & 0xF]; + } + /* Map 21-bit characters to '\U00xxxxxx' */ + else { + *odata++ = 'U'; + *odata++ = Py_hexdigits[(ch >> 28) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 24) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 20) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 16) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 12) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 8) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 4) & 0xF]; + *odata++ = Py_hexdigits[ch & 0xF]; + } + } + /* Copy characters as-is */ + else { + *odata++ = ch; + } + } + } + *odata = quote; +} diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 92db31f1e498f9..eb37b478cc4de1 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -899,6 +899,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -909,6 +910,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -919,6 +921,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -12336,24 +12339,17 @@ unicode_removesuffix_impl(PyObject *self, PyObject *suffix) static PyObject * unicode_repr(PyObject *unicode) { - PyObject *repr; - Py_ssize_t isize; - Py_ssize_t osize, squote, dquote, i, o; - Py_UCS4 max, quote; - int ikind, okind, unchanged; - const void *idata; - void *odata; - - isize = PyUnicode_GET_LENGTH(unicode); - idata = PyUnicode_DATA(unicode); + Py_ssize_t isize = PyUnicode_GET_LENGTH(unicode); + const void *idata = PyUnicode_DATA(unicode); /* Compute length of output, quote characters, and maximum character */ - osize = 0; - max = 127; - squote = dquote = 0; - ikind = PyUnicode_KIND(unicode); - for (i = 0; i < isize; i++) { + Py_ssize_t osize = 0; + Py_UCS4 maxch = 127; + Py_ssize_t squote = 0; + Py_ssize_t dquote = 0; + int ikind = PyUnicode_KIND(unicode); + for (Py_ssize_t i = 0; i < isize; i++) { Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); Py_ssize_t incr = 1; switch (ch) { @@ -12369,7 +12365,7 @@ unicode_repr(PyObject *unicode) else if (ch < 0x7f) ; else if (Py_UNICODE_ISPRINTABLE(ch)) - max = ch > max ? ch : max; + maxch = (ch > maxch) ? ch : maxch; else if (ch < 0x100) incr = 4; /* \xHH */ else if (ch < 0x10000) @@ -12385,10 +12381,10 @@ unicode_repr(PyObject *unicode) osize += incr; } - quote = '\''; - unchanged = (osize == isize); + Py_UCS4 quote = '\''; + int changed = (osize != isize); if (squote) { - unchanged = 0; + changed = 1; if (dquote) /* Both squote and dquote present. Use squote, and escape them */ @@ -12398,99 +12394,35 @@ unicode_repr(PyObject *unicode) } osize += 2; /* quotes */ - repr = PyUnicode_New(osize, max); + PyObject *repr = PyUnicode_New(osize, maxch); if (repr == NULL) return NULL; - okind = PyUnicode_KIND(repr); - odata = PyUnicode_DATA(repr); + int okind = PyUnicode_KIND(repr); + void *odata = PyUnicode_DATA(repr); + + if (!changed) { + PyUnicode_WRITE(okind, odata, 0, quote); - PyUnicode_WRITE(okind, odata, 0, quote); - PyUnicode_WRITE(okind, odata, osize-1, quote); - if (unchanged) { _PyUnicode_FastCopyCharacters(repr, 1, unicode, 0, isize); + + PyUnicode_WRITE(okind, odata, osize-1, quote); } else { - for (i = 0, o = 1; i < isize; i++) { - Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); - - /* Escape quotes and backslashes */ - if ((ch == quote) || (ch == '\\')) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, ch); - continue; - } - - /* Map special whitespace to '\t', \n', '\r' */ - if (ch == '\t') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 't'); - } - else if (ch == '\n') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'n'); - } - else if (ch == '\r') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'r'); - } - - /* Map non-printable US ASCII to '\xhh' */ - else if (ch < ' ' || ch == 0x7F) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'x'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); - } - - /* Copy ASCII characters as-is */ - else if (ch < 0x7F) { - PyUnicode_WRITE(okind, odata, o++, ch); - } - - /* Non-ASCII characters */ - else { - /* Map Unicode whitespace and control characters - (categories Z* and C* except ASCII space) - */ - if (!Py_UNICODE_ISPRINTABLE(ch)) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - /* Map 8-bit characters to '\xhh' */ - if (ch <= 0xff) { - PyUnicode_WRITE(okind, odata, o++, 'x'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); - } - /* Map 16-bit characters to '\uxxxx' */ - else if (ch <= 0xffff) { - PyUnicode_WRITE(okind, odata, o++, 'u'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); - } - /* Map 21-bit characters to '\U00xxxxxx' */ - else { - PyUnicode_WRITE(okind, odata, o++, 'U'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 28) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 24) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 20) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 16) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); - } - } - /* Copy characters as-is */ - else { - PyUnicode_WRITE(okind, odata, o++, ch); - } - } + switch (okind) { + case PyUnicode_1BYTE_KIND: + ucs1lib_repr(unicode, quote, odata); + break; + case PyUnicode_2BYTE_KIND: + ucs2lib_repr(unicode, quote, odata); + break; + default: + assert(okind == PyUnicode_4BYTE_KIND); + ucs4lib_repr(unicode, quote, odata); } } - /* Closing quote already added at the beginning */ + assert(_PyUnicode_CheckConsistency(repr, 1)); return repr; } diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 12010f0e9c0549..4623f2c8d671bd 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -167,6 +167,7 @@ def clean_lines(text): Objects/stringlib/find.h Objects/stringlib/fastsearch.h Objects/stringlib/partition.h Objects/stringlib/fastsearch.h Objects/stringlib/replace.h Objects/stringlib/fastsearch.h +Objects/stringlib/repr.h Objects/stringlib/fastsearch.h Objects/stringlib/split.h Objects/stringlib/fastsearch.h # @end=tsv@ From 6ec371223dff4da7719039e271f35a16a5b861c6 Mon Sep 17 00:00:00 2001 From: Steven Troxler Date: Tue, 28 May 2024 10:18:57 -0700 Subject: [PATCH 240/903] gh-119581: Add a test of InitVar with name shadowing (#119582) --- Lib/test/test_dataclasses/__init__.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index ea49596eaa4d96..04dd9f3265bb33 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -1317,6 +1317,29 @@ def __post_init__(self, init_base, init_derived): c = C(10, 11, 50, 51) self.assertEqual(vars(c), {'x': 21, 'y': 101}) + def test_init_var_name_shadowing(self): + # Because dataclasses rely exclusively on `__annotations__` for + # handling InitVar and `__annotations__` preserves shadowed definitions, + # you can actually shadow an InitVar with a method or property. + # + # This only works when there is no default value; `dataclasses` uses the + # actual name (which will be bound to the shadowing method) for default + # values. + @dataclass + class C: + shadowed: InitVar[int] + _shadowed: int = field(init=False) + + def __post_init__(self, shadowed): + self._shadowed = shadowed * 2 + + @property + def shadowed(self): + return self._shadowed * 3 + + c = C(5) + self.assertEqual(c.shadowed, 30) + def test_default_factory(self): # Test a factory that returns a new list. @dataclass From a89fc26f4ac90156a1b108135f92dcf7698190f2 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 28 May 2024 19:27:52 +0200 Subject: [PATCH 241/903] gh-117398: gh-119655: datetime: Init static state once & don't free it (GH-119662) - While datetime uses global state, only initialize it once. - While `capi` is static, don't free it (thanks @neonene in https://github.com/python/cpython/pull/119641/files#r1616710048) --- Modules/_datetimemodule.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 271b37dfcded6c..54383ca5177368 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -48,6 +48,9 @@ typedef struct { /* The interned Unix epoch datetime instance */ PyObject *epoch; + + /* While we use a global state, we ensure it's only initialized once */ + int initialized; } datetime_state; static datetime_state _datetime_global_state; @@ -6841,6 +6844,12 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) static int init_state(datetime_state *st) { + // While datetime uses global module "state", we unly initialize it once. + // The PyLong objects created here (once per process) are not decref'd. + if (st->initialized) { + return 0; + } + st->date_type = &PyDateTime_DateType; st->datetime_type = &PyDateTime_DateTimeType; st->delta_type = &PyDateTime_DeltaType; @@ -6893,6 +6902,9 @@ init_state(datetime_state *st) if (st->epoch == NULL) { return -1; } + + st->initialized = 1; + return 0; } @@ -7005,12 +7017,8 @@ _datetime_exec(PyObject *module) goto error; } PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, NULL); - if (capsule == NULL) { - PyMem_Free(capi); - goto error; - } + // (capsule == NULL) is handled by PyModule_Add if (PyModule_Add(module, "datetime_CAPI", capsule) < 0) { - PyMem_Free(capi); goto error; } From ae11d68ab90324a3359699ca13fcf9a229966713 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 28 May 2024 11:04:08 -0700 Subject: [PATCH 242/903] gh-117865: Defer import of re in ast (#119546) This is used only by ast.get_source_segment(), so it seems sensible to avoid importing it. Co-authored-by: Alex Waygood --- Lib/ast.py | 9 +++++++-- .../2024-05-25-07-25-07.gh-issue-117865.1A0Xpi.rst | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-25-07-25-07.gh-issue-117865.1A0Xpi.rst diff --git a/Lib/ast.py b/Lib/ast.py index bc6c3347787d61..fb4d21b87d8bd0 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -25,7 +25,6 @@ :license: Python License. """ import sys -import re from _ast import * from contextlib import contextmanager, nullcontext from enum import IntEnum, auto, _simple_enum @@ -325,12 +324,18 @@ def get_docstring(node, clean=True): return text -_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") +_line_pattern = None def _splitlines_no_ff(source, maxlines=None): """Split a string into lines ignoring form feed and other chars. This mimics how the Python parser splits source code. """ + global _line_pattern + if _line_pattern is None: + # lazily computed to speedup import time of `ast` + import re + _line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") + lines = [] for lineno, match in enumerate(_line_pattern.finditer(source), 1): if maxlines is not None and lineno > maxlines: diff --git a/Misc/NEWS.d/next/Library/2024-05-25-07-25-07.gh-issue-117865.1A0Xpi.rst b/Misc/NEWS.d/next/Library/2024-05-25-07-25-07.gh-issue-117865.1A0Xpi.rst new file mode 100644 index 00000000000000..48cd390d1bb128 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-07-25-07.gh-issue-117865.1A0Xpi.rst @@ -0,0 +1,2 @@ +Improve the import time of the :mod:`ast` module by deferring the import of +:mod:`re`. Patch by Jelle Zijlstra. From 6b240c2308a044e38623900ccb8fa58c3549d4ae Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 28 May 2024 21:12:58 +0300 Subject: [PATCH 243/903] gh-119011: `type.__type_params__` now return an empty tuple (#119296) --- Lib/test/test_functools.py | 8 ++++++++ Lib/test/test_type_params.py | 5 +++++ .../2024-05-21-09-46-51.gh-issue-119011.WOe3bu.rst | 2 ++ Objects/typeobject.c | 4 ++++ 4 files changed, 19 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-21-09-46-51.gh-issue-119011.WOe3bu.rst diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 4a9a7313712f60..26701ea8b4daf9 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -710,6 +710,14 @@ def wrapper(): self.assertTrue(wrapper.__doc__.startswith('max(')) self.assertEqual(wrapper.__annotations__, {}) + def test_update_type_wrapper(self): + def wrapper(*args): pass + + functools.update_wrapper(wrapper, type) + self.assertEqual(wrapper.__name__, 'type') + self.assertEqual(wrapper.__annotations__, {}) + self.assertEqual(wrapper.__type_params__, ()) + class TestWraps(TestUpdateWrapper): diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index aa9d4088bd7ccc..bf1a34b9fc82b3 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -563,6 +563,11 @@ class C[T, U]: self.assertIs(T, C.Alias.__type_params__[0]) self.assertIs(U, C.__type_params__[1]) + def test_type_special_case(self): + # https://github.com/python/cpython/issues/119011 + self.assertEqual(type.__type_params__, ()) + self.assertEqual(object.__type_params__, ()) + def make_base(arg): class Base: diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-21-09-46-51.gh-issue-119011.WOe3bu.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-09-46-51.gh-issue-119011.WOe3bu.rst new file mode 100644 index 00000000000000..0083c18da13278 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-21-09-46-51.gh-issue-119011.WOe3bu.rst @@ -0,0 +1,2 @@ +Fixes ``type.__type_params__`` to return an empty tuple instead of a +descriptor. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 11f9c570ac4971..9d849d83082ccd 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1848,6 +1848,10 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) static PyObject * type_get_type_params(PyTypeObject *type, void *context) { + if (type == &PyType_Type) { + return PyTuple_New(0); + } + PyObject *params; if (PyDict_GetItemRef(lookup_tp_dict(type), &_Py_ID(__type_params__), ¶ms) == 0) { return PyTuple_New(0); From ae9140f32a1630838374f1af402291d4649a0be0 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 28 May 2024 20:05:38 +0100 Subject: [PATCH 244/903] gh-119676: remove several pseudo instructions which are use only in codegen (#119677) --- Include/internal/pycore_opcode_metadata.h | 32 ++++++----------------- Include/opcode_ids.h | 14 ++++------ Lib/_opcode_metadata.py | 14 ++++------ Python/bytecodes.c | 16 ------------ Python/compile.c | 18 ++++--------- 5 files changed, 23 insertions(+), 71 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 6da4702b2cb2ee..4aedfa89e906ea 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -20,10 +20,6 @@ extern "C" { #define IS_PSEUDO_INSTR(OP) ( \ ((OP) == LOAD_CLOSURE) || \ ((OP) == STORE_FAST_MAYBE_NULL) || \ - ((OP) == LOAD_SUPER_METHOD) || \ - ((OP) == LOAD_ZERO_SUPER_METHOD) || \ - ((OP) == LOAD_ZERO_SUPER_ATTR) || \ - ((OP) == LOAD_METHOD) || \ ((OP) == JUMP) || \ ((OP) == JUMP_NO_INTERRUPT) || \ ((OP) == SETUP_FINALLY) || \ @@ -912,7 +908,7 @@ enum InstructionFormat { }; #define IS_VALID_OPCODE(OP) \ - (((OP) >= 0) && ((OP) < 268) && \ + (((OP) >= 0) && ((OP) < 264) && \ (_PyOpcode_opcode_metadata[(OP)].valid_entry)) #define HAS_ARG_FLAG (1) @@ -961,9 +957,9 @@ struct opcode_metadata { int16_t flags; }; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[268]; +extern const struct opcode_metadata _PyOpcode_opcode_metadata[264]; #ifdef NEED_OPCODE_METADATA -const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { +const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1178,10 +1174,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, - [LOAD_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ZERO_SUPER_ATTR] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ZERO_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, @@ -1364,9 +1356,9 @@ _PyOpcode_macro_expansion[256] = { }; #endif // NEED_OPCODE_METADATA -extern const char *_PyOpcode_OpName[268]; +extern const char *_PyOpcode_OpName[264]; #ifdef NEED_OPCODE_METADATA -const char *_PyOpcode_OpName[268] = { +const char *_PyOpcode_OpName[264] = { [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", [BEFORE_WITH] = "BEFORE_WITH", [BINARY_OP] = "BINARY_OP", @@ -1516,14 +1508,10 @@ const char *_PyOpcode_OpName[268] = { [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [LOAD_LOCALS] = "LOAD_LOCALS", - [LOAD_METHOD] = "LOAD_METHOD", [LOAD_NAME] = "LOAD_NAME", [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", - [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", - [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", [MAKE_CELL] = "MAKE_CELL", [MAKE_FUNCTION] = "MAKE_FUNCTION", [MAP_ADD] = "MAP_ADD", @@ -1887,15 +1875,11 @@ const uint8_t _PyOpcode_Deopt[256] = { struct pseudo_targets { uint8_t targets[3]; }; -extern const struct pseudo_targets _PyOpcode_PseudoTargets[12]; +extern const struct pseudo_targets _PyOpcode_PseudoTargets[8]; #ifdef NEED_OPCODE_METADATA -const struct pseudo_targets _PyOpcode_PseudoTargets[12] = { +const struct pseudo_targets _PyOpcode_PseudoTargets[8] = { [LOAD_CLOSURE-256] = { { LOAD_FAST, 0, 0 } }, [STORE_FAST_MAYBE_NULL-256] = { { STORE_FAST, 0, 0 } }, - [LOAD_SUPER_METHOD-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, - [LOAD_ZERO_SUPER_METHOD-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, - [LOAD_ZERO_SUPER_ATTR-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, - [LOAD_METHOD-256] = { { LOAD_ATTR, 0, 0 } }, [JUMP-256] = { { JUMP_FORWARD, JUMP_BACKWARD, 0 } }, [JUMP_NO_INTERRUPT-256] = { { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0 } }, [SETUP_FINALLY-256] = { { NOP, 0, 0 } }, @@ -1907,7 +1891,7 @@ const struct pseudo_targets _PyOpcode_PseudoTargets[12] = { #endif // NEED_OPCODE_METADATA static inline bool is_pseudo_target(int pseudo, int target) { - if (pseudo < 256 || pseudo >= 268) { + if (pseudo < 256 || pseudo >= 264) { return false; } for (int i = 0; _PyOpcode_PseudoTargets[pseudo-256].targets[i]; i++) { diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 6a608651d1e81d..5b37de25703560 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -225,15 +225,11 @@ extern "C" { #define JUMP 256 #define JUMP_NO_INTERRUPT 257 #define LOAD_CLOSURE 258 -#define LOAD_METHOD 259 -#define LOAD_SUPER_METHOD 260 -#define LOAD_ZERO_SUPER_ATTR 261 -#define LOAD_ZERO_SUPER_METHOD 262 -#define POP_BLOCK 263 -#define SETUP_CLEANUP 264 -#define SETUP_FINALLY 265 -#define SETUP_WITH 266 -#define STORE_FAST_MAYBE_NULL 267 +#define POP_BLOCK 259 +#define SETUP_CLEANUP 260 +#define SETUP_FINALLY 261 +#define SETUP_WITH 262 +#define STORE_FAST_MAYBE_NULL 263 #define HAVE_ARGUMENT 43 #define MIN_INSTRUMENTED_OPCODE 236 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 4da924bd250821..c5d1c79fe6b043 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -328,15 +328,11 @@ 'JUMP': 256, 'JUMP_NO_INTERRUPT': 257, 'LOAD_CLOSURE': 258, - 'LOAD_METHOD': 259, - 'LOAD_SUPER_METHOD': 260, - 'LOAD_ZERO_SUPER_ATTR': 261, - 'LOAD_ZERO_SUPER_METHOD': 262, - 'POP_BLOCK': 263, - 'SETUP_CLEANUP': 264, - 'SETUP_FINALLY': 265, - 'SETUP_WITH': 266, - 'STORE_FAST_MAYBE_NULL': 267, + 'POP_BLOCK': 259, + 'SETUP_CLEANUP': 260, + 'SETUP_FINALLY': 261, + 'SETUP_WITH': 262, + 'STORE_FAST_MAYBE_NULL': 263, } HAVE_ARGUMENT = 43 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b48f913b456064..025fed35686ca6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1812,18 +1812,6 @@ dummy_func( macro(LOAD_SUPER_ATTR) = _SPECIALIZE_LOAD_SUPER_ATTR + _LOAD_SUPER_ATTR; - pseudo(LOAD_SUPER_METHOD) = { - LOAD_SUPER_ATTR, - }; - - pseudo(LOAD_ZERO_SUPER_METHOD) = { - LOAD_SUPER_ATTR, - }; - - pseudo(LOAD_ZERO_SUPER_ATTR) = { - LOAD_SUPER_ATTR, - }; - inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) { assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type); @@ -1925,10 +1913,6 @@ dummy_func( unused/8 + _LOAD_ATTR; - pseudo(LOAD_METHOD) = { - LOAD_ATTR, - }; - op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); diff --git a/Python/compile.c b/Python/compile.c index cdc5b26ec70066..e6efae33eb45e4 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -723,9 +723,6 @@ stack_effect(int opcode, int oparg, int jump) case JUMP_NO_INTERRUPT: return 0; - case EXIT_INIT_CHECK: - return -1; - /* Exception handling pseudo-instructions */ case SETUP_FINALLY: /* 0 in the normal flow. @@ -746,12 +743,6 @@ stack_effect(int opcode, int oparg, int jump) return -1; case LOAD_CLOSURE: return 1; - case LOAD_METHOD: - return 1; - case LOAD_SUPER_METHOD: - case LOAD_ZERO_SUPER_METHOD: - case LOAD_ZERO_SUPER_ATTR: - return -1; default: return PY_INVALID_STACK_EFFECT; } @@ -997,6 +988,11 @@ compiler_addop_o(struct compiler_unit *u, location loc, return codegen_addop_i(u->u_instr_sequence, opcode, arg, loc); } +#define LOAD_METHOD -1 +#define LOAD_SUPER_METHOD -2 +#define LOAD_ZERO_SUPER_ATTR -3 +#define LOAD_ZERO_SUPER_METHOD -4 + static int compiler_addop_name(struct compiler_unit *u, location loc, int opcode, PyObject *dict, PyObject *o) @@ -1014,7 +1010,6 @@ compiler_addop_name(struct compiler_unit *u, location loc, arg <<= 1; } if (opcode == LOAD_METHOD) { - assert(is_pseudo_target(LOAD_METHOD, LOAD_ATTR)); opcode = LOAD_ATTR; arg <<= 1; arg |= 1; @@ -1024,18 +1019,15 @@ compiler_addop_name(struct compiler_unit *u, location loc, arg |= 2; } if (opcode == LOAD_SUPER_METHOD) { - assert(is_pseudo_target(LOAD_SUPER_METHOD, LOAD_SUPER_ATTR)); opcode = LOAD_SUPER_ATTR; arg <<= 2; arg |= 3; } if (opcode == LOAD_ZERO_SUPER_ATTR) { - assert(is_pseudo_target(LOAD_ZERO_SUPER_ATTR, LOAD_SUPER_ATTR)); opcode = LOAD_SUPER_ATTR; arg <<= 2; } if (opcode == LOAD_ZERO_SUPER_METHOD) { - assert(is_pseudo_target(LOAD_ZERO_SUPER_METHOD, LOAD_SUPER_ATTR)); opcode = LOAD_SUPER_ATTR; arg <<= 2; arg |= 1; From d87b0151062e36e67f9e42e1595fba5bf23a485c Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 28 May 2024 21:17:49 +0200 Subject: [PATCH 245/903] gh-119118: Fix performance regression in tokenize module (#119615) * gh-119118: Fix performance regression in tokenize module - Cache line object to avoid creating a Unicode object for all of the tokens in the same line. - Speed up byte offset to column offset conversion by using the smallest buffer possible to measure the difference. Co-authored-by: Pablo Galindo --- ...-05-28-12-15-03.gh-issue-119118.FMKz1F.rst | 2 + Parser/pegen.c | 25 +++++++++++ Parser/pegen.h | 1 + Python/Python-tokenize.c | 44 +++++++++++++++++-- 4 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-28-12-15-03.gh-issue-119118.FMKz1F.rst diff --git a/Misc/NEWS.d/next/Library/2024-05-28-12-15-03.gh-issue-119118.FMKz1F.rst b/Misc/NEWS.d/next/Library/2024-05-28-12-15-03.gh-issue-119118.FMKz1F.rst new file mode 100644 index 00000000000000..3cf61662fe7767 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-28-12-15-03.gh-issue-119118.FMKz1F.rst @@ -0,0 +1,2 @@ +Fix performance regression in the :mod:`tokenize` module by caching the ``line`` +token attribute and calculating the column offset more efficiently. diff --git a/Parser/pegen.c b/Parser/pegen.c index 3d3e64559403b1..2955eab2dac7c4 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -18,6 +18,31 @@ _PyPegen_interactive_exit(Parser *p) return NULL; } +Py_ssize_t +_PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset) +{ + const char *data = PyUnicode_AsUTF8(line); + + Py_ssize_t len = 0; + while (col_offset < end_col_offset) { + Py_UCS4 ch = data[col_offset]; + if (ch < 0x80) { + col_offset += 1; + } else if ((ch & 0xe0) == 0xc0) { + col_offset += 2; + } else if ((ch & 0xf0) == 0xe0) { + col_offset += 3; + } else if ((ch & 0xf8) == 0xf0) { + col_offset += 4; + } else { + PyErr_SetString(PyExc_ValueError, "Invalid UTF-8 sequence"); + return -1; + } + len++; + } + return len; +} + Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char* str, Py_ssize_t col_offset) { diff --git a/Parser/pegen.h b/Parser/pegen.h index 57b45a54c36c57..32c64e7774b878 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -148,6 +148,7 @@ int _PyPegen_fill_token(Parser *p); expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); +Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset); Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset); Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char*, Py_ssize_t col_offset); diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 41e8107e205b46..9cc4b45de49f0b 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -32,6 +32,11 @@ typedef struct { PyObject_HEAD struct tok_state *tok; int done; + + /* Needed to cache line for performance */ + PyObject *last_line; + Py_ssize_t last_lineno; + Py_ssize_t byte_col_offset_diff; } tokenizeriterobject; /*[clinic input] @@ -68,6 +73,11 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, self->tok->tok_extra_tokens = 1; } self->done = 0; + + self->last_line = NULL; + self->byte_col_offset_diff = 0; + self->last_lineno = 0; + return (PyObject *)self; } @@ -210,7 +220,18 @@ tokenizeriter_next(tokenizeriterobject *it) if (size >= 1 && it->tok->implicit_newline) { size -= 1; } - line = PyUnicode_DecodeUTF8(line_start, size, "replace"); + + if (it->tok->lineno != it->last_lineno) { + // Line has changed since last token, so we fetch the new line and cache it + // in the iter object. + Py_XDECREF(it->last_line); + line = PyUnicode_DecodeUTF8(line_start, size, "replace"); + it->last_line = line; + it->byte_col_offset_diff = 0; + } else { + // Line hasn't changed so we reuse the cached one. + line = it->last_line; + } } if (line == NULL) { Py_DECREF(str); @@ -219,13 +240,28 @@ tokenizeriter_next(tokenizeriterobject *it) Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno; Py_ssize_t end_lineno = it->tok->lineno; + it->last_lineno = lineno; + Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; + Py_ssize_t byte_offset = -1; if (token.start != NULL && token.start >= line_start) { - col_offset = _PyPegen_byte_offset_to_character_offset(line, token.start - line_start); + byte_offset = token.start - line_start; + col_offset = byte_offset - it->byte_col_offset_diff; } if (token.end != NULL && token.end >= it->tok->line_start) { - end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, token.end - it->tok->line_start); + Py_ssize_t end_byte_offset = token.end - it->tok->line_start; + if (lineno == end_lineno) { + // If the whole token is at the same line, we can just use the token.start + // buffer for figuring out the new column offset, since using line is not + // performant for very long lines. + Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); + end_col_offset = col_offset + token_col_offset; + it->byte_col_offset_diff += token.end - token.start - token_col_offset; + } else { + end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset); + it->byte_col_offset_diff += end_byte_offset - end_col_offset; + } } if (it->tok->tok_extra_tokens) { @@ -262,7 +298,7 @@ tokenizeriter_next(tokenizeriterobject *it) } } - result = Py_BuildValue("(iN(nn)(nn)N)", type, str, lineno, col_offset, end_lineno, end_col_offset, line); + result = Py_BuildValue("(iN(nn)(nn)O)", type, str, lineno, col_offset, end_lineno, end_col_offset, line); exit: _PyToken_Free(&token); if (type == ENDMARKER) { From cfcc054dee87a5a3f18c99d83a2957bf3487de1d Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 28 May 2024 12:45:11 -0700 Subject: [PATCH 246/903] GH-119476: Split _CHECK_FUNCTION_VERSION out of _CHECK_FUNCTION_EXACT_ARGS (GH-119510) --- Include/internal/pycore_opcode_metadata.h | 6 +++--- Python/bytecodes.c | 7 ++++--- Python/executor_cases.c.h | 10 +--------- Python/generated_cases.c.h | 20 +++++++++++++++----- Python/optimizer_bytecodes.c | 3 +-- Python/optimizer_cases.c.h | 2 -- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 4aedfa89e906ea..7f18b07a9eacc3 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1182,7 +1182,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { }; #endif -#define MAX_UOP_PER_EXPANSION 8 +#define MAX_UOP_PER_EXPANSION 9 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1212,7 +1212,7 @@ _PyOpcode_macro_expansion[256] = { [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 9, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BOUND_METHOD_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, 0, 0 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, @@ -1227,7 +1227,7 @@ _PyOpcode_macro_expansion[256] = { [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, 0, 0 }, { _CALL_NON_PY_GENERAL, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 7, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_PY_GENERAL] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_STR_1] = { .nuops = 2, .uops = { { _CALL_STR_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_TUPLE_1] = { .nuops = 2, .uops = { { _CALL_TUPLE_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 025fed35686ca6..8b12da0918520b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3261,10 +3261,9 @@ dummy_func( DEOPT_IF(tstate->interp->eval_frame); } - op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - EXIT_IF(!PyFunction_Check(callable)); + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + assert(PyFunction_Check(callable)); PyFunctionObject *func = (PyFunctionObject *)callable; - EXIT_IF(func->func_version != func_version); PyCodeObject *code = (PyCodeObject *)func->func_code; EXIT_IF(code->co_argcount != oparg + (self_or_null != NULL)); } @@ -3308,6 +3307,7 @@ dummy_func( _CHECK_PEP_523 + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + _INIT_CALL_BOUND_METHOD_EXACT_ARGS + + _CHECK_FUNCTION_VERSION + _CHECK_FUNCTION_EXACT_ARGS + _CHECK_STACK_SPACE + _INIT_CALL_PY_EXACT_ARGS + @@ -3317,6 +3317,7 @@ dummy_func( macro(CALL_PY_EXACT_ARGS) = unused/1 + // Skip over the counter _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION + _CHECK_FUNCTION_EXACT_ARGS + _CHECK_STACK_SPACE + _INIT_CALL_PY_EXACT_ARGS + diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a3d7af250316e3..a00b3821912394 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3240,16 +3240,8 @@ oparg = CURRENT_OPARG(); self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - if (!PyFunction_Check(callable)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyFunction_Check(callable)); PyFunctionObject *func = (PyFunctionObject *)callable; - if (func->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } PyCodeObject *code = (PyCodeObject *)func->func_code; if (code->co_argcount != oparg + (self_or_null != NULL)) { UOP_STAT_INC(uopcode, miss); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 07d9965b299e12..b897c38cc474b2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -943,14 +943,19 @@ stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization Py_DECREF(callable); } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = self; + // _CHECK_FUNCTION_VERSION callable = func; { uint32_t func_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = stack_pointer[-1 - oparg]; + { + assert(PyFunction_Check(callable)); + PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } @@ -1859,8 +1864,8 @@ next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *self_or_null; PyObject *callable; + PyObject *self_or_null; PyObject **args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -1868,14 +1873,19 @@ { DEOPT_IF(tstate->interp->eval_frame, CALL); } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = stack_pointer[-1 - oparg]; + // _CHECK_FUNCTION_VERSION callable = stack_pointer[-2 - oparg]; { uint32_t func_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = stack_pointer[-1 - oparg]; + { + assert(PyFunction_Check(callable)); + PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e5c982befb2411..a2cb4c0b2c5192 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -519,10 +519,9 @@ dummy_func(void) { self = sym_new_not_null(ctx); } - op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { sym_set_type(callable, &PyFunction_Type); (void)self_or_null; - (void)func_version; } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 30ed011a83eb6e..4d5c8032cb33cf 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1537,10 +1537,8 @@ _Py_UopsSymbol *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)this_instr->operand; sym_set_type(callable, &PyFunction_Type); (void)self_or_null; - (void)func_version; break; } From 5cd3ffd6b70a63dbae4d34a1b4db033e19184159 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 28 May 2024 12:47:54 -0700 Subject: [PATCH 247/903] GH-119258: Handle STORE_ATTR_WITH_HINT in tier two (GH-119481) --- Include/internal/pycore_dict.h | 2 +- Include/internal/pycore_opcode_metadata.h | 3 +- Include/internal/pycore_uop_ids.h | 30 ++++----- Include/internal/pycore_uop_metadata.h | 4 ++ Python/bytecodes.c | 12 ++-- Python/executor_cases.c.h | 62 +++++++++++++++++- Python/generated_cases.c.h | 80 ++++++++++++----------- Python/optimizer_cases.c.h | 5 +- 8 files changed, 137 insertions(+), 61 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 8d8d3748edaea8..cfe837b1e2b3ab 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -246,7 +246,7 @@ dict_next_version(PyInterpreterState *interp) ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) #endif -void +PyAPI_FUNC(void) _PyDict_SendEvent(int watcher_bits, PyDict_WatchEvent event, PyDictObject *mp, diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 7f18b07a9eacc3..f805be04985ef2 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1142,7 +1142,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG }, - [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1325,6 +1325,7 @@ _PyOpcode_macro_expansion[256] = { [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, + [STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } }, [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } }, [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d36b172f57ec68..19e2b823ed0140 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -249,28 +249,28 @@ extern "C" { #define _STORE_ATTR 436 #define _STORE_ATTR_INSTANCE_VALUE 437 #define _STORE_ATTR_SLOT 438 -#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT +#define _STORE_ATTR_WITH_HINT 439 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 439 -#define _STORE_FAST_0 440 -#define _STORE_FAST_1 441 -#define _STORE_FAST_2 442 -#define _STORE_FAST_3 443 -#define _STORE_FAST_4 444 -#define _STORE_FAST_5 445 -#define _STORE_FAST_6 446 -#define _STORE_FAST_7 447 +#define _STORE_FAST 440 +#define _STORE_FAST_0 441 +#define _STORE_FAST_1 442 +#define _STORE_FAST_2 443 +#define _STORE_FAST_3 444 +#define _STORE_FAST_4 445 +#define _STORE_FAST_5 446 +#define _STORE_FAST_6 447 +#define _STORE_FAST_7 448 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 448 +#define _STORE_SUBSCR 449 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 449 -#define _TO_BOOL 450 +#define _TIER2_RESUME_CHECK 450 +#define _TO_BOOL 451 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -280,13 +280,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 451 +#define _UNPACK_SEQUENCE 452 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 451 +#define MAX_UOP_ID 452 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f555824fc8f3b3..78f0eafaa32042 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -152,6 +152,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, [_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = 0, + [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR_SLOT] = 0, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, @@ -474,6 +475,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_ATTR_WITH_HINT] = "_STORE_ATTR_WITH_HINT", [_STORE_DEREF] = "_STORE_DEREF", [_STORE_FAST] = "_STORE_FAST", [_STORE_FAST_0] = "_STORE_FAST_0", @@ -780,6 +782,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _STORE_ATTR_INSTANCE_VALUE: return 2; + case _STORE_ATTR_WITH_HINT: + return 2; case _STORE_ATTR_SLOT: return 2; case _COMPARE_OP: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8b12da0918520b..274c5c22447e4c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2119,11 +2119,8 @@ dummy_func( _GUARD_DORV_NO_DICT + _STORE_ATTR_INSTANCE_VALUE; - inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + op(_STORE_ATTR_WITH_HINT, (hint/1, value, owner --)) { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner); DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); @@ -2158,6 +2155,11 @@ dummy_func( Py_DECREF(owner); } + macro(STORE_ATTR_WITH_HINT) = + unused/1 + + _GUARD_TYPE_VERSION + + _STORE_ATTR_WITH_HINT; + op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a00b3821912394..e862364cb23e7a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2279,7 +2279,67 @@ break; } - /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 because it has unused cache entries */ + case _STORE_ATTR_WITH_HINT: { + PyObject *owner; + PyObject *value; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + uint16_t hint = (uint16_t)CURRENT_OPERAND(); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner); + if (dict == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *old_value; + uint64_t new_version; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + old_value = ep->me_value; + if (old_value == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + old_value = ep->me_value; + if (old_value == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; + } + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + _PyObject_GC_TRACK(dict); + } + /* PEP 509 */ + dict->ma_version_tag = new_version; + Py_DECREF(owner); + stack_pointer += -2; + break; + } case _STORE_ATTR_SLOT: { PyObject *owner; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b897c38cc474b2..4402787d96f12e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5578,46 +5578,52 @@ PyObject *owner; PyObject *value; /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint16_t hint = read_u16(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner); - DEOPT_IF(dict == NULL, STORE_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *old_value; - uint64_t new_version; - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - Py_DECREF(old_value); - STAT_INC(STORE_ATTR, hit); - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { - _PyObject_GC_TRACK(dict); + // _STORE_ATTR_WITH_HINT + value = stack_pointer[-2]; + { + uint16_t hint = read_u16(&this_instr[4].cache); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner); + DEOPT_IF(dict == NULL, STORE_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + PyObject *old_value; + uint64_t new_version; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; + } + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + _PyObject_GC_TRACK(dict); + } + /* PEP 509 */ + dict->ma_version_tag = new_version; + Py_DECREF(owner); } - /* PEP 509 */ - dict->ma_version_tag = new_version; - Py_DECREF(owner); stack_pointer += -2; DISPATCH(); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4d5c8032cb33cf..1b76f1480b4f11 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1079,7 +1079,10 @@ break; } - /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 */ + case _STORE_ATTR_WITH_HINT: { + stack_pointer += -2; + break; + } case _STORE_ATTR_SLOT: { stack_pointer += -2; From 606be663622c6784aed4ffa55b877adbd6fe8e54 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 28 May 2024 22:05:19 +0200 Subject: [PATCH 248/903] gh-119538: Add missing expat build dependencies (#119647) xmltok_impl.c and xmltok_ns.c are _included_ in xmltok.c by the C pre-processor. --- Makefile.pre.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index a80d9334ba5134..b259537eae6ecf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -631,7 +631,9 @@ LIBEXPAT_HEADERS= \ Modules/expat/utf8tab.h \ Modules/expat/xmlrole.h \ Modules/expat/xmltok.h \ - Modules/expat/xmltok_impl.h + Modules/expat/xmltok_impl.h \ + Modules/expat/xmltok_impl.c \ + Modules/expat/xmltok_ns.c ########################################################################## # hashlib's HACL* library From 548a11d5cf1dbb32d86ce0c045130c77f50c1427 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 May 2024 18:42:23 -0400 Subject: [PATCH 249/903] gh-117398: Convert datetime.IsoCalendarDate To A Heap Type (gh-119637) This is the only static type in the module that we will not keep static. --- Modules/_datetimemodule.c | 82 ++++++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 22 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 54383ca5177368..b72a5d3c70b92a 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -26,14 +26,18 @@ #endif typedef struct { + /* Static types exposed by the datetime C-API. */ PyTypeObject *date_type; PyTypeObject *datetime_type; PyTypeObject *delta_type; - PyTypeObject *isocalendar_date_type; PyTypeObject *time_type; PyTypeObject *tzinfo_type; + /* Exposed indirectly via TimeZone_UTC. */ PyTypeObject *timezone_type; + /* Other module classes. */ + PyTypeObject *isocalendar_date_type; + /* Conversion factors. */ PyObject *us_per_ms; // 1_000 PyObject *us_per_second; // 1_000_000 @@ -3460,17 +3464,40 @@ static PyMethodDef iso_calendar_date_methods[] = { {NULL, NULL}, }; -static PyTypeObject PyDateTime_IsoCalendarDateType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "datetime.IsoCalendarDate", - .tp_basicsize = sizeof(PyDateTime_IsoCalendarDate), - .tp_repr = (reprfunc) iso_calendar_date_repr, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = iso_calendar_date__doc__, - .tp_methods = iso_calendar_date_methods, - .tp_getset = iso_calendar_date_getset, - // .tp_base = &PyTuple_Type, // filled in PyInit__datetime - .tp_new = iso_calendar_date_new, +static int +iso_calendar_date_traverse(PyDateTime_IsoCalendarDate *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return PyTuple_Type.tp_traverse((PyObject *)self, visit, arg); +} + +static void +iso_calendar_date_dealloc(PyDateTime_IsoCalendarDate *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyTuple_Type.tp_dealloc((PyObject *)self); // delegate GC-untrack as well + Py_DECREF(tp); +} + +static PyType_Slot isocal_slots[] = { + {Py_tp_repr, iso_calendar_date_repr}, + {Py_tp_doc, (void *)iso_calendar_date__doc__}, + {Py_tp_methods, iso_calendar_date_methods}, + {Py_tp_getset, iso_calendar_date_getset}, + {Py_tp_new, iso_calendar_date_new}, + {Py_tp_dealloc, iso_calendar_date_dealloc}, + {Py_tp_traverse, iso_calendar_date_traverse}, + {0, NULL}, +}; + +static PyType_Spec isocal_spec = { + .name = "datetime.IsoCalendarDate", + .basicsize = sizeof(PyDateTime_IsoCalendarDate), + .flags = (Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = isocal_slots, }; /*[clinic input] @@ -6842,7 +6869,7 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) } static int -init_state(datetime_state *st) +init_state(datetime_state *st, PyTypeObject *PyDateTime_IsoCalendarDateType) { // While datetime uses global module "state", we unly initialize it once. // The PyLong objects created here (once per process) are not decref'd. @@ -6850,14 +6877,17 @@ init_state(datetime_state *st) return 0; } + /* Static types exposed by the C-API. */ st->date_type = &PyDateTime_DateType; st->datetime_type = &PyDateTime_DateTimeType; st->delta_type = &PyDateTime_DeltaType; - st->isocalendar_date_type = &PyDateTime_IsoCalendarDateType; st->time_type = &PyDateTime_TimeType; st->tzinfo_type = &PyDateTime_TZInfoType; st->timezone_type = &PyDateTime_TimeZoneType; + /* Per-module heap types. */ + st->isocalendar_date_type = PyDateTime_IsoCalendarDateType; + st->us_per_ms = PyLong_FromLong(1000); if (st->us_per_ms == NULL) { return -1; @@ -6914,11 +6944,10 @@ _datetime_exec(PyObject *module) // `&...` is not a constant expression according to a strict reading // of C standards. Fill tp_base at run-time rather than statically. // See https://bugs.python.org/issue40777 - PyDateTime_IsoCalendarDateType.tp_base = &PyTuple_Type; PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; - PyTypeObject *types[] = { + PyTypeObject *capi_types[] = { &PyDateTime_DateType, &PyDateTime_DateTimeType, &PyDateTime_TimeType, @@ -6927,18 +6956,27 @@ _datetime_exec(PyObject *module) &PyDateTime_TimeZoneType, }; - for (size_t i = 0; i < Py_ARRAY_LENGTH(types); i++) { - if (PyModule_AddType(module, types[i]) < 0) { + for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { + if (PyModule_AddType(module, capi_types[i]) < 0) { goto error; } } - if (PyType_Ready(&PyDateTime_IsoCalendarDateType) < 0) { - goto error; - } +#define CREATE_TYPE(VAR, SPEC, BASE) \ + do { \ + VAR = (PyTypeObject *)PyType_FromModuleAndSpec( \ + module, SPEC, (PyObject *)BASE); \ + if (VAR == NULL) { \ + goto error; \ + } \ + } while (0) + + PyTypeObject *PyDateTime_IsoCalendarDateType = NULL; + CREATE_TYPE(PyDateTime_IsoCalendarDateType, &isocal_spec, &PyTuple_Type); +#undef CREATE_TYPE datetime_state *st = get_datetime_state(); - if (init_state(st) < 0) { + if (init_state(st, PyDateTime_IsoCalendarDateType) < 0) { goto error; } From a8e35e8ebad8c3bb44d14968aa05d1acbc028247 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 28 May 2024 17:05:18 -0700 Subject: [PATCH 250/903] gh-119443: Turn off from __future__ import annotations in REPL (#119493) --- Lib/_pyrepl/simple_interact.py | 2 +- Lib/test/test_pyrepl/test_interact.py | 9 +++++++++ .../2024-05-23-22-29-59.gh-issue-119443.KAGz6S.rst | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-23-22-29-59.gh-issue-119443.KAGz6S.rst diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 975533a425be23..b5f182ebfe710b 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -100,7 +100,7 @@ def runsource(self, source, filename="", symbol="single"): the_symbol = symbol if stmt is last_stmt else "exec" item = wrapper([stmt]) try: - code = compile(item, filename, the_symbol) + code = compile(item, filename, the_symbol, dont_inherit=True) except (OverflowError, ValueError): self.showsyntaxerror(filename) return False diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index 10e34045bcf92d..6ebd51fe14dd62 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -94,3 +94,12 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self): with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: console.runsource(source) mock_showsyntaxerror.assert_called_once() + + def test_no_active_future(self): + console = InteractiveColoredConsole() + source = "x: int = 1; print(__annotations__)" + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertEqual(f.getvalue(), "{'x': }\n") diff --git a/Misc/NEWS.d/next/Library/2024-05-23-22-29-59.gh-issue-119443.KAGz6S.rst b/Misc/NEWS.d/next/Library/2024-05-23-22-29-59.gh-issue-119443.KAGz6S.rst new file mode 100644 index 00000000000000..4470c566a37d88 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-23-22-29-59.gh-issue-119443.KAGz6S.rst @@ -0,0 +1,2 @@ +The interactive REPL no longer runs with ``from __future__ import +annotations`` enabled. Patch by Jelle Zijlstra. From c0faade891e6ccb61137041fe10cc05e5fa8d534 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 29 May 2024 09:56:44 +0300 Subject: [PATCH 251/903] gh-119704: Fix reference leak in the ``Python/Python-tokenize.c`` (#119705) --- Python/Python-tokenize.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 9cc4b45de49f0b..09fad18b5b4df7 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -311,6 +311,7 @@ static void tokenizeriter_dealloc(tokenizeriterobject *it) { PyTypeObject *tp = Py_TYPE(it); + Py_XDECREF(it->last_line); _PyTokenizer_Free(it->tok); tp->tp_free(it); Py_DECREF(tp); From 86d1a1aa8841ea182eaf434ae6b942b3e93f58db Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 29 May 2024 09:57:50 +0300 Subject: [PATCH 252/903] gh-119555: catch SyntaxError from compile() in the InteractiveColoredConsole (#119557) --- Lib/_pyrepl/simple_interact.py | 2 +- Lib/test/test_pyrepl/test_interact.py | 8 ++++++++ .../2024-05-25-20-15-26.gh-issue-119555.mvHbEL.rst | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-25-20-15-26.gh-issue-119555.mvHbEL.rst diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index b5f182ebfe710b..11e831c1d6c5d4 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -101,7 +101,7 @@ def runsource(self, source, filename="", symbol="single"): item = wrapper([stmt]) try: code = compile(item, filename, the_symbol, dont_inherit=True) - except (OverflowError, ValueError): + except (OverflowError, ValueError, SyntaxError): self.showsyntaxerror(filename) return False diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index 6ebd51fe14dd62..4d01ea7620109d 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -94,6 +94,14 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self): with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: console.runsource(source) mock_showsyntaxerror.assert_called_once() + source = dedent("""\ + match 1: + case {0: _, 0j: _}: + pass + """) + with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: + console.runsource(source) + mock_showsyntaxerror.assert_called_once() def test_no_active_future(self): console = InteractiveColoredConsole() diff --git a/Misc/NEWS.d/next/Library/2024-05-25-20-15-26.gh-issue-119555.mvHbEL.rst b/Misc/NEWS.d/next/Library/2024-05-25-20-15-26.gh-issue-119555.mvHbEL.rst new file mode 100644 index 00000000000000..e16cb28b471a7a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-20-15-26.gh-issue-119555.mvHbEL.rst @@ -0,0 +1,2 @@ +Catch :exc:`SyntaxError` from :func:`compile` in the runsource() method of +the InteractiveColoredConsole. Patch by Sergey B Kirpichev. From cd11ff12ac55f37d38b5ef08c143c78f07da5717 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 29 May 2024 10:51:19 +0300 Subject: [PATCH 253/903] gh-119613: Use C99+ functions instead of Py_IS_NAN/INFINITY/FINITE (#119619) --- Modules/_decimal/_decimal.c | 4 +- Modules/_json.c | 2 +- Modules/cmathmodule.c | 72 +++++++++---------- Modules/mathmodule.c | 140 ++++++++++++++++++------------------ Objects/complexobject.c | 8 +-- Objects/floatobject.c | 30 ++++---- Objects/longobject.c | 4 +- Python/ast_unparse.c | 2 +- Python/bltinmodule.c | 4 +- Python/pyhash.c | 4 +- Python/pystrtod.c | 6 +- Python/pytime.c | 6 +- 12 files changed, 140 insertions(+), 142 deletions(-) diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 2daa24c823a542..94a2cc2c8e5f8a 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -2425,12 +2425,12 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, } sign = (copysign(1.0, x) == 1.0) ? 0 : 1; - if (Py_IS_NAN(x) || Py_IS_INFINITY(x)) { + if (isnan(x) || isinf(x)) { dec = PyDecType_New(type); if (dec == NULL) { return NULL; } - if (Py_IS_NAN(x)) { + if (isnan(x)) { /* decimal.py calls repr(float(+-nan)), * which always gives a positive result. */ mpd_setspecial(MPD(dec), MPD_POS, MPD_NAN); diff --git a/Modules/_json.c b/Modules/_json.c index e33ef1f5eea92f..c7fe1561bb1018 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1326,7 +1326,7 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) { /* Return the JSON representation of a PyFloat. */ double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { + if (!isfinite(i)) { if (!s->allow_nan) { PyErr_Format( PyExc_ValueError, diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index d901b350bc5343..bf86a211bcb188 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -117,7 +117,7 @@ enum special_types { static enum special_types special_type(double d) { - if (Py_IS_FINITE(d)) { + if (isfinite(d)) { if (d != 0) { if (copysign(1., d) == 1.) return ST_POS; @@ -131,7 +131,7 @@ special_type(double d) return ST_NZERO; } } - if (Py_IS_NAN(d)) + if (isnan(d)) return ST_NAN; if (copysign(1., d) == 1.) return ST_PINF; @@ -139,11 +139,11 @@ special_type(double d) return ST_NINF; } -#define SPECIAL_VALUE(z, table) \ - if (!Py_IS_FINITE((z).real) || !Py_IS_FINITE((z).imag)) { \ - errno = 0; \ - return table[special_type((z).real)] \ - [special_type((z).imag)]; \ +#define SPECIAL_VALUE(z, table) \ + if (!isfinite((z).real) || !isfinite((z).imag)) { \ + errno = 0; \ + return table[special_type((z).real)] \ + [special_type((z).imag)]; \ } #define P Py_MATH_PI @@ -329,10 +329,10 @@ cmath_atan_impl(PyObject *module, Py_complex z) static double c_atan2(Py_complex z) { - if (Py_IS_NAN(z.real) || Py_IS_NAN(z.imag)) + if (isnan(z.real) || isnan(z.imag)) return Py_NAN; - if (Py_IS_INFINITY(z.imag)) { - if (Py_IS_INFINITY(z.real)) { + if (isinf(z.imag)) { + if (isinf(z.real)) { if (copysign(1., z.real) == 1.) /* atan2(+-inf, +inf) == +-pi/4 */ return copysign(0.25*Py_MATH_PI, z.imag); @@ -343,7 +343,7 @@ c_atan2(Py_complex z) /* atan2(+-inf, x) == +-pi/2 for finite x */ return copysign(0.5*Py_MATH_PI, z.imag); } - if (Py_IS_INFINITY(z.real) || z.imag == 0.) { + if (isinf(z.real) || z.imag == 0.) { if (copysign(1., z.real) == 1.) /* atan2(+-y, +inf) = atan2(+-0, +x) = +-0. */ return copysign(0., z.imag); @@ -448,8 +448,8 @@ cmath_cosh_impl(PyObject *module, Py_complex z) double x_minus_one; /* special treatment for cosh(+/-inf + iy) if y is not a NaN */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) && + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -466,7 +466,7 @@ cmath_cosh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN */ - if (Py_IS_INFINITY(z.imag) && !Py_IS_NAN(z.real)) + if (isinf(z.imag) && !isnan(z.real)) errno = EDOM; else errno = 0; @@ -484,7 +484,7 @@ cmath_cosh_impl(PyObject *module, Py_complex z) r.imag = sin(z.imag) * sinh(z.real); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -509,8 +509,8 @@ cmath_exp_impl(PyObject *module, Py_complex z) Py_complex r; double l; - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -527,9 +527,9 @@ cmath_exp_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN and not -infinity */ - if (Py_IS_INFINITY(z.imag) && - (Py_IS_FINITE(z.real) || - (Py_IS_INFINITY(z.real) && z.real > 0))) + if (isinf(z.imag) && + (isfinite(z.real) || + (isinf(z.real) && z.real > 0))) errno = EDOM; else errno = 0; @@ -546,7 +546,7 @@ cmath_exp_impl(PyObject *module, Py_complex z) r.imag = l*sin(z.imag); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -686,8 +686,8 @@ cmath_sinh_impl(PyObject *module, Py_complex z) /* special treatment for sinh(+/-inf + iy) if y is finite and nonzero */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -704,7 +704,7 @@ cmath_sinh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN */ - if (Py_IS_INFINITY(z.imag) && !Py_IS_NAN(z.real)) + if (isinf(z.imag) && !isnan(z.real)) errno = EDOM; else errno = 0; @@ -720,7 +720,7 @@ cmath_sinh_impl(PyObject *module, Py_complex z) r.imag = sin(z.imag) * cosh(z.real); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -856,8 +856,8 @@ cmath_tanh_impl(PyObject *module, Py_complex z) /* special treatment for tanh(+/-inf + iy) if y is finite and nonzero */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = 1.0; @@ -876,7 +876,7 @@ cmath_tanh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if z.imag is +/-infinity and z.real is finite */ - if (Py_IS_INFINITY(z.imag) && Py_IS_FINITE(z.real)) + if (isinf(z.imag) && isfinite(z.real)) errno = EDOM; else errno = 0; @@ -1030,11 +1030,11 @@ cmath_rect_impl(PyObject *module, double r, double phi) errno = 0; /* deal with special values */ - if (!Py_IS_FINITE(r) || !Py_IS_FINITE(phi)) { + if (!isfinite(r) || !isfinite(phi)) { /* if r is +/-infinity and phi is finite but nonzero then result is (+-INF +-INF i), but we need to compute cos(phi) and sin(phi) to figure out the signs. */ - if (Py_IS_INFINITY(r) && (Py_IS_FINITE(phi) + if (isinf(r) && (isfinite(phi) && (phi != 0.))) { if (r > 0) { z.real = copysign(INF, cos(phi)); @@ -1051,7 +1051,7 @@ cmath_rect_impl(PyObject *module, double r, double phi) } /* need to set errno = EDOM if r is a nonzero number and phi is infinite */ - if (r != 0. && !Py_IS_NAN(r) && Py_IS_INFINITY(phi)) + if (r != 0. && !isnan(r) && isinf(phi)) errno = EDOM; else errno = 0; @@ -1085,7 +1085,7 @@ static PyObject * cmath_isfinite_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=ac76611e2c774a36 input=848e7ee701895815]*/ { - return PyBool_FromLong(Py_IS_FINITE(z.real) && Py_IS_FINITE(z.imag)); + return PyBool_FromLong(isfinite(z.real) && isfinite(z.imag)); } /*[clinic input] @@ -1098,7 +1098,7 @@ static PyObject * cmath_isnan_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=e7abf6e0b28beab7 input=71799f5d284c9baf]*/ { - return PyBool_FromLong(Py_IS_NAN(z.real) || Py_IS_NAN(z.imag)); + return PyBool_FromLong(isnan(z.real) || isnan(z.imag)); } /*[clinic input] @@ -1111,8 +1111,7 @@ static PyObject * cmath_isinf_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=502a75a79c773469 input=363df155c7181329]*/ { - return PyBool_FromLong(Py_IS_INFINITY(z.real) || - Py_IS_INFINITY(z.imag)); + return PyBool_FromLong(isinf(z.real) || isinf(z.imag)); } /*[clinic input] @@ -1167,8 +1166,7 @@ cmath_isclose_impl(PyObject *module, Py_complex a, Py_complex b, above. */ - if (Py_IS_INFINITY(a.real) || Py_IS_INFINITY(a.imag) || - Py_IS_INFINITY(b.real) || Py_IS_INFINITY(b.imag)) { + if (isinf(a.real) || isinf(a.imag) || isinf(b.real) || isinf(b.imag)) { return 0; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index a79694730a8e4e..6defa973da0952 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -237,7 +237,7 @@ m_sinpi(double x) double y, r; int n; /* this function should only ever be called for finite arguments */ - assert(Py_IS_FINITE(x)); + assert(isfinite(x)); y = fmod(fabs(x), 2.0); n = (int)round(2.0*y); assert(0 <= n && n <= 4); @@ -396,8 +396,8 @@ m_tgamma(double x) double absx, r, y, z, sqrtpow; /* special cases */ - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x) || x > 0.0) + if (!isfinite(x)) { + if (isnan(x) || x > 0.0) return x; /* tgamma(nan) = nan, tgamma(inf) = inf */ else { errno = EDOM; @@ -424,7 +424,7 @@ m_tgamma(double x) /* tiny arguments: tgamma(x) ~ 1/x for x near 0 */ if (absx < 1e-20) { r = 1.0/x; - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } @@ -481,7 +481,7 @@ m_tgamma(double x) r *= sqrtpow; } } - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } @@ -498,8 +498,8 @@ m_lgamma(double x) double absx; /* special cases */ - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x)) + if (!isfinite(x)) { + if (isnan(x)) return x; /* lgamma(nan) = nan */ else return Py_HUGE_VAL; /* lgamma(+-inf) = +inf */ @@ -530,7 +530,7 @@ m_lgamma(double x) if (x < 0.0) /* Use reflection formula to get value for negative x. */ r = logpi - log(fabs(m_sinpi(absx))) - log(absx) - r; - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } @@ -546,10 +546,10 @@ m_lgamma(double x) static double m_atan2(double y, double x) { - if (Py_IS_NAN(x) || Py_IS_NAN(y)) + if (isnan(x) || isnan(y)) return Py_NAN; - if (Py_IS_INFINITY(y)) { - if (Py_IS_INFINITY(x)) { + if (isinf(y)) { + if (isinf(x)) { if (copysign(1., x) == 1.) /* atan2(+-inf, +inf) == +-pi/4 */ return copysign(0.25*Py_MATH_PI, y); @@ -560,7 +560,7 @@ m_atan2(double y, double x) /* atan2(+-inf, x) == +-pi/2 for finite x */ return copysign(0.5*Py_MATH_PI, y); } - if (Py_IS_INFINITY(x) || y == 0.) { + if (isinf(x) || y == 0.) { if (copysign(1., x) == 1.) /* atan2(+-y, +inf) = atan2(+-0, +x) = +-0. */ return copysign(0., y); @@ -580,7 +580,7 @@ static double m_remainder(double x, double y) { /* Deal with most common case first. */ - if (Py_IS_FINITE(x) && Py_IS_FINITE(y)) { + if (isfinite(x) && isfinite(y)) { double absx, absy, c, m, r; if (y == 0.0) { @@ -653,16 +653,16 @@ m_remainder(double x, double y) } /* Special values. */ - if (Py_IS_NAN(x)) { + if (isnan(x)) { return x; } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return y; } - if (Py_IS_INFINITY(x)) { + if (isinf(x)) { return Py_NAN; } - assert(Py_IS_INFINITY(y)); + assert(isinf(y)); return x; } @@ -677,7 +677,7 @@ m_remainder(double x, double y) static double m_log(double x) { - if (Py_IS_FINITE(x)) { + if (isfinite(x)) { if (x > 0.0) return log(x); errno = EDOM; @@ -686,7 +686,7 @@ m_log(double x) else return Py_NAN; /* log(-ve) = nan */ } - else if (Py_IS_NAN(x)) + else if (isnan(x)) return x; /* log(nan) = nan */ else if (x > 0.0) return x; /* log(inf) = inf */ @@ -709,8 +709,8 @@ m_log(double x) static double m_log2(double x) { - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x)) + if (!isfinite(x)) { + if (isnan(x)) return x; /* log2(nan) = nan */ else if (x > 0.0) return x; /* log2(+inf) = +inf */ @@ -736,7 +736,7 @@ m_log2(double x) static double m_log10(double x) { - if (Py_IS_FINITE(x)) { + if (isfinite(x)) { if (x > 0.0) return log10(x); errno = EDOM; @@ -745,7 +745,7 @@ m_log10(double x) else return Py_NAN; /* log10(-ve) = nan */ } - else if (Py_IS_NAN(x)) + else if (isnan(x)) return x; /* log10(nan) = nan */ else if (x > 0.0) return x; /* log10(inf) = inf */ @@ -966,12 +966,12 @@ math_1(PyObject *arg, double (*func) (double), int can_overflow) return NULL; errno = 0; r = (*func)(x); - if (Py_IS_NAN(r) && !Py_IS_NAN(x)) { + if (isnan(r) && !isnan(x)) { PyErr_SetString(PyExc_ValueError, "math domain error"); /* invalid arg */ return NULL; } - if (Py_IS_INFINITY(r) && Py_IS_FINITE(x)) { + if (isinf(r) && isfinite(x)) { if (can_overflow) PyErr_SetString(PyExc_OverflowError, "math range error"); /* overflow */ @@ -980,7 +980,7 @@ math_1(PyObject *arg, double (*func) (double), int can_overflow) "math domain error"); /* singularity */ return NULL; } - if (Py_IS_FINITE(r) && errno && is_error(r)) + if (isfinite(r) && errno && is_error(r)) /* this branch unnecessary on most platforms */ return NULL; @@ -1049,14 +1049,14 @@ math_2(PyObject *const *args, Py_ssize_t nargs, } errno = 0; r = (*func)(x, y); - if (Py_IS_NAN(r)) { - if (!Py_IS_NAN(x) && !Py_IS_NAN(y)) + if (isnan(r)) { + if (!isnan(x) && !isnan(y)) errno = EDOM; else errno = 0; } - else if (Py_IS_INFINITY(r)) { - if (Py_IS_FINITE(x) && Py_IS_FINITE(y)) + else if (isinf(r)) { + if (isfinite(x) && isfinite(y)) errno = ERANGE; else errno = 0; @@ -1403,17 +1403,17 @@ math_fsum(PyObject *module, PyObject *seq) n = i; /* ps[i:] = [x] */ if (x != 0.0) { - if (! Py_IS_FINITE(x)) { + if (! isfinite(x)) { /* a nonfinite x could arise either as a result of intermediate overflow, or as a result of a nan or inf in the summands */ - if (Py_IS_FINITE(xsave)) { + if (isfinite(xsave)) { PyErr_SetString(PyExc_OverflowError, "intermediate overflow in fsum"); goto _fsum_error; } - if (Py_IS_INFINITY(xsave)) + if (isinf(xsave)) inf_sum += xsave; special_sum += xsave; /* reset partials */ @@ -1427,7 +1427,7 @@ math_fsum(PyObject *module, PyObject *seq) } if (special_sum != 0.0) { - if (Py_IS_NAN(inf_sum)) + if (isnan(inf_sum)) PyErr_SetString(PyExc_ValueError, "-inf + inf in fsum"); else @@ -2108,7 +2108,7 @@ math_frexp_impl(PyObject *module, double x) int i; /* deal with special cases directly, to sidestep platform differences */ - if (Py_IS_NAN(x) || Py_IS_INFINITY(x) || !x) { + if (isnan(x) || isinf(x) || !x) { i = 0; } else { @@ -2153,7 +2153,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) return NULL; } - if (x == 0. || !Py_IS_FINITE(x)) { + if (x == 0. || !isfinite(x)) { /* NaNs, zeros and infinities are returned unchanged */ r = x; errno = 0; @@ -2168,7 +2168,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) } else { errno = 0; r = ldexp(x, (int)exp); - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; } @@ -2196,9 +2196,9 @@ math_modf_impl(PyObject *module, double x) double y; /* some platforms don't do the right thing for NaNs and infinities, so we take care of special cases directly. */ - if (Py_IS_INFINITY(x)) + if (isinf(x)) return Py_BuildValue("(dd)", copysign(0., x), x); - else if (Py_IS_NAN(x)) + else if (isnan(x)) return Py_BuildValue("(dd)", x, x); errno = 0; @@ -2341,19 +2341,19 @@ math_fma_impl(PyObject *module, double x, double y, double z) double r = fma(x, y, z); /* Fast path: if we got a finite result, we're done. */ - if (Py_IS_FINITE(r)) { + if (isfinite(r)) { return PyFloat_FromDouble(r); } /* Non-finite result. Raise an exception if appropriate, else return r. */ - if (Py_IS_NAN(r)) { - if (!Py_IS_NAN(x) && !Py_IS_NAN(y) && !Py_IS_NAN(z)) { + if (isnan(r)) { + if (!isnan(x) && !isnan(y) && !isnan(z)) { /* NaN result from non-NaN inputs. */ PyErr_SetString(PyExc_ValueError, "invalid operation in fma"); return NULL; } } - else if (Py_IS_FINITE(x) && Py_IS_FINITE(y) && Py_IS_FINITE(z)) { + else if (isfinite(x) && isfinite(y) && isfinite(z)) { /* Infinite result from finite inputs. */ PyErr_SetString(PyExc_OverflowError, "overflow in fma"); return NULL; @@ -2381,12 +2381,12 @@ math_fmod_impl(PyObject *module, double x, double y) { double r; /* fmod(x, +/-Inf) returns x for finite x. */ - if (Py_IS_INFINITY(y) && Py_IS_FINITE(x)) + if (isinf(y) && isfinite(x)) return PyFloat_FromDouble(x); errno = 0; r = fmod(x, y); - if (Py_IS_NAN(r)) { - if (!Py_IS_NAN(x) && !Py_IS_NAN(y)) + if (isnan(r)) { + if (!isnan(x) && !isnan(y)) errno = EDOM; else errno = 0; @@ -2508,7 +2508,7 @@ vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) int max_e; Py_ssize_t i; - if (Py_IS_INFINITY(max)) { + if (isinf(max)) { return max; } if (found_nan) { @@ -2530,7 +2530,7 @@ vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) assert(max * scale < 1.0); for (i=0 ; i < n ; i++) { x = vec[i]; - assert(Py_IS_FINITE(x) && fabs(x) <= max); + assert(isfinite(x) && fabs(x) <= max); x *= scale; // lossless scaling assert(fabs(x) < 1.0); pr = dl_mul(x, x); // lossless squaring @@ -2620,7 +2620,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) ASSIGN_DOUBLE(qx, item, error_exit); x = fabs(px - qx); diffs[i] = x; - found_nan |= Py_IS_NAN(x); + found_nan |= isnan(x); if (x > max) { max = x; } @@ -2673,7 +2673,7 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) ASSIGN_DOUBLE(x, item, error_exit); x = fabs(x); coordinates[i] = x; - found_nan |= Py_IS_NAN(x); + found_nan |= isnan(x); if (x > max) { max = x; } @@ -2976,14 +2976,14 @@ math_pow_impl(PyObject *module, double x, double y) /* deal directly with IEEE specials, to cope with problems on various platforms whose semantics don't exactly match C99 */ r = 0.; /* silence compiler warning */ - if (!Py_IS_FINITE(x) || !Py_IS_FINITE(y)) { + if (!isfinite(x) || !isfinite(y)) { errno = 0; - if (Py_IS_NAN(x)) + if (isnan(x)) r = y == 0. ? 1. : x; /* NaN**0 = 1 */ - else if (Py_IS_NAN(y)) + else if (isnan(y)) r = x == 1. ? 1. : y; /* 1**NaN = 1 */ - else if (Py_IS_INFINITY(x)) { - odd_y = Py_IS_FINITE(y) && fmod(fabs(y), 2.0) == 1.0; + else if (isinf(x)) { + odd_y = isfinite(y) && fmod(fabs(y), 2.0) == 1.0; if (y > 0.) r = odd_y ? x : fabs(x); else if (y == 0.) @@ -2992,7 +2992,7 @@ math_pow_impl(PyObject *module, double x, double y) r = odd_y ? copysign(0., x) : 0.; } else { - assert(Py_IS_INFINITY(y)); + assert(isinf(y)); if (fabs(x) == 1.0) r = 1.; else if (y > 0. && fabs(x) > 1.0) @@ -3010,8 +3010,8 @@ math_pow_impl(PyObject *module, double x, double y) r = pow(x, y); /* a NaN result should arise only from (-ve)**(finite non-integer); in this case we want to raise ValueError. */ - if (!Py_IS_FINITE(r)) { - if (Py_IS_NAN(r)) { + if (!isfinite(r)) { + if (isnan(r)) { errno = EDOM; } /* @@ -3019,7 +3019,7 @@ math_pow_impl(PyObject *module, double x, double y) (A) (+/-0.)**negative (-> divide-by-zero) (B) overflow of x**y with x and y finite */ - else if (Py_IS_INFINITY(r)) { + else if (isinf(r)) { if (x == 0.) errno = EDOM; else @@ -3085,7 +3085,7 @@ static PyObject * math_isfinite_impl(PyObject *module, double x) /*[clinic end generated code: output=8ba1f396440c9901 input=46967d254812e54a]*/ { - return PyBool_FromLong((long)Py_IS_FINITE(x)); + return PyBool_FromLong((long)isfinite(x)); } @@ -3102,7 +3102,7 @@ static PyObject * math_isnan_impl(PyObject *module, double x) /*[clinic end generated code: output=f537b4d6df878c3e input=935891e66083f46a]*/ { - return PyBool_FromLong((long)Py_IS_NAN(x)); + return PyBool_FromLong((long)isnan(x)); } @@ -3119,7 +3119,7 @@ static PyObject * math_isinf_impl(PyObject *module, double x) /*[clinic end generated code: output=9f00cbec4de7b06b input=32630e4212cf961f]*/ { - return PyBool_FromLong((long)Py_IS_INFINITY(x)); + return PyBool_FromLong((long)isinf(x)); } @@ -3176,7 +3176,7 @@ math_isclose_impl(PyObject *module, double a, double b, double rel_tol, above. */ - if (Py_IS_INFINITY(a) || Py_IS_INFINITY(b)) { + if (isinf(a) || isinf(b)) { return 0; } @@ -3926,10 +3926,10 @@ math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) Bug fixed in bos.adt.libm 7.2.2.0 by APAR IV95512. */ return PyFloat_FromDouble(y); } - if (Py_IS_NAN(x)) { + if (isnan(x)) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return PyFloat_FromDouble(y); } #endif @@ -3975,10 +3975,10 @@ math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) if (usteps == 0) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(x)) { + if (isnan(x)) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return PyFloat_FromDouble(y); } @@ -4044,16 +4044,16 @@ static double math_ulp_impl(PyObject *module, double x) /*[clinic end generated code: output=f5207867a9384dd4 input=31f9bfbbe373fcaa]*/ { - if (Py_IS_NAN(x)) { + if (isnan(x)) { return x; } x = fabs(x); - if (Py_IS_INFINITY(x)) { + if (isinf(x)) { return x; } double inf = Py_INFINITY; double x2 = nextafter(x, inf); - if (Py_IS_INFINITY(x2)) { + if (isinf(x2)) { /* special case: x is the largest positive representable float */ x2 = nextafter(x, -inf); return x - x2; diff --git a/Objects/complexobject.c b/Objects/complexobject.c index d8b0e84da5df4a..943c5ccabfd5c4 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -188,16 +188,16 @@ _Py_c_abs(Py_complex z) /* sets errno = ERANGE on overflow; otherwise errno = 0 */ double result; - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { + if (!isfinite(z.real) || !isfinite(z.imag)) { /* C99 rules: if either the real or the imaginary part is an infinity, return infinity, even if the other part is a NaN. */ - if (Py_IS_INFINITY(z.real)) { + if (isinf(z.real)) { result = fabs(z.real); errno = 0; return result; } - if (Py_IS_INFINITY(z.imag)) { + if (isinf(z.imag)) { result = fabs(z.imag); errno = 0; return result; @@ -207,7 +207,7 @@ _Py_c_abs(Py_complex z) return Py_NAN; } result = hypot(z.real, z.imag); - if (!Py_IS_FINITE(result)) + if (!isfinite(result)) errno = ERANGE; else errno = 0; diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 96227f2cf7d76f..a5bf356cc9c7f0 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -418,7 +418,7 @@ float_richcompare(PyObject *v, PyObject *w, int op) if (PyFloat_Check(w)) j = PyFloat_AS_DOUBLE(w); - else if (!Py_IS_FINITE(i)) { + else if (!isfinite(i)) { if (PyLong_Check(w)) /* If i is an infinity, its magnitude exceeds any * finite integer, so it doesn't matter which int we @@ -749,13 +749,13 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) if (iw == 0) { /* v**0 is 1, even 0**0 */ return PyFloat_FromDouble(1.0); } - if (Py_IS_NAN(iv)) { /* nan**w = nan, unless w == 0 */ + if (isnan(iv)) { /* nan**w = nan, unless w == 0 */ return PyFloat_FromDouble(iv); } - if (Py_IS_NAN(iw)) { /* v**nan = nan, unless v == 1; 1**nan = 1 */ + if (isnan(iw)) { /* v**nan = nan, unless v == 1; 1**nan = 1 */ return PyFloat_FromDouble(iv == 1.0 ? 1.0 : iw); } - if (Py_IS_INFINITY(iw)) { + if (isinf(iw)) { /* v**inf is: 0.0 if abs(v) < 1; 1.0 if abs(v) == 1; inf if * abs(v) > 1 (including case where v infinite) * @@ -770,7 +770,7 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) else return PyFloat_FromDouble(0.0); } - if (Py_IS_INFINITY(iv)) { + if (isinf(iv)) { /* (+-inf)**w is: inf for w positive, 0 for w negative; in * both cases, we need to add the appropriate sign if w is * an odd integer. @@ -885,7 +885,7 @@ float_is_integer_impl(PyObject *self) if (x == -1.0 && PyErr_Occurred()) return NULL; - if (!Py_IS_FINITE(x)) + if (!isfinite(x)) Py_RETURN_FALSE; errno = 0; o = (floor(x) == x) ? Py_True : Py_False; @@ -1021,7 +1021,7 @@ double_round(double x, int ndigits) { } y = (x*pow1)*pow2; /* if y overflows, then rounded value is exactly x */ - if (!Py_IS_FINITE(y)) + if (!isfinite(y)) return PyFloat_FromDouble(x); } else { @@ -1041,7 +1041,7 @@ double_round(double x, int ndigits) { z *= pow1; /* if computation resulted in overflow, raise OverflowError */ - if (!Py_IS_FINITE(z)) { + if (!isfinite(z)) { PyErr_SetString(PyExc_OverflowError, "overflow occurred during round"); return NULL; @@ -1089,7 +1089,7 @@ float___round___impl(PyObject *self, PyObject *o_ndigits) return NULL; /* nans and infinities round to themselves */ - if (!Py_IS_FINITE(x)) + if (!isfinite(x)) return PyFloat_FromDouble(x); /* Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x @@ -1237,7 +1237,7 @@ float_hex_impl(PyObject *self) CONVERT_TO_DOUBLE(self, x); - if (Py_IS_NAN(x) || Py_IS_INFINITY(x)) + if (isnan(x) || isinf(x)) return float_repr((PyFloatObject *)self); if (x == 0.0) { @@ -1570,12 +1570,12 @@ float_as_integer_ratio_impl(PyObject *self) CONVERT_TO_DOUBLE(self, self_double); - if (Py_IS_INFINITY(self_double)) { + if (isinf(self_double)) { PyErr_SetString(PyExc_OverflowError, "cannot convert Infinity to integer ratio"); return NULL; } - if (Py_IS_NAN(self_double)) { + if (isnan(self_double)) { PyErr_SetString(PyExc_ValueError, "cannot convert NaN to integer ratio"); return NULL; @@ -2060,12 +2060,12 @@ PyFloat_Pack2(double x, char *data, int le) e = 0; bits = 0; } - else if (Py_IS_INFINITY(x)) { + else if (isinf(x)) { sign = (x < 0.0); e = 0x1f; bits = 0; } - else if (Py_IS_NAN(x)) { + else if (isnan(x)) { /* There are 2046 distinct half-precision NaNs (1022 signaling and 1024 quiet), but there are only two quiet NaNs that don't arise by quieting a signaling NaN; we get those by setting the topmost bit @@ -2234,7 +2234,7 @@ PyFloat_Pack4(double x, char *data, int le) float y = (float)x; int i, incr = 1; - if (Py_IS_INFINITY(y) && !Py_IS_INFINITY(x)) + if (isinf(y) && !isinf(x)) goto Overflow; unsigned char s[sizeof(float)]; diff --git a/Objects/longobject.c b/Objects/longobject.c index b0456a311409bf..2dc2cb7a47b460 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -401,12 +401,12 @@ PyLong_FromDouble(double dval) double frac; int i, ndig, expo, neg; neg = 0; - if (Py_IS_INFINITY(dval)) { + if (isinf(dval)) { PyErr_SetString(PyExc_OverflowError, "cannot convert float infinity to integer"); return NULL; } - if (Py_IS_NAN(dval)) { + if (isnan(dval)) { PyErr_SetString(PyExc_ValueError, "cannot convert float NaN to integer"); return NULL; diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index 8aff045101cc72..27c34008d4d5e6 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -79,7 +79,7 @@ append_repr(_PyUnicodeWriter *writer, PyObject *obj) return -1; } - if ((PyFloat_CheckExact(obj) && Py_IS_INFINITY(PyFloat_AS_DOUBLE(obj))) || + if ((PyFloat_CheckExact(obj) && isinf(PyFloat_AS_DOUBLE(obj))) || PyComplex_CheckExact(obj)) { PyInterpreterState *interp = _PyInterpreterState_GET(); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d192d5be751cfc..2a02d8161591c6 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2640,7 +2640,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) /* Avoid losing the sign on a negative result, and don't let adding the compensation convert an infinite or overflowed sum to a NaN. */ - if (c && Py_IS_FINITE(c)) { + if (c && isfinite(c)) { f_result += c; } return PyFloat_FromDouble(f_result); @@ -2672,7 +2672,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) continue; } } - if (c && Py_IS_FINITE(c)) { + if (c && isfinite(c)) { f_result += c; } result = PyFloat_FromDouble(f_result); diff --git a/Python/pyhash.c b/Python/pyhash.c index 5263622ff3126d..4145d9ef4fd7ef 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -90,8 +90,8 @@ _Py_HashDouble(PyObject *inst, double v) double m; Py_uhash_t x, y; - if (!Py_IS_FINITE(v)) { - if (Py_IS_INFINITY(v)) + if (!isfinite(v)) { + if (isinf(v)) return v > 0 ? _PyHASH_INF : -_PyHASH_INF; else return PyObject_GenericHash(inst); diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 16bf06f0e6cca2..5c8be0447ace4b 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -842,7 +842,7 @@ char * PyOS_double_to_string(double val, */ - if (Py_IS_NAN(val) || Py_IS_INFINITY(val)) + if (isnan(val) || isinf(val)) /* 3 for 'inf'/'nan', 1 for sign, 1 for '\0' */ bufsize = 5; else { @@ -860,10 +860,10 @@ char * PyOS_double_to_string(double val, } /* Handle nan and inf. */ - if (Py_IS_NAN(val)) { + if (isnan(val)) { strcpy(buf, "nan"); t = Py_DTST_NAN; - } else if (Py_IS_INFINITY(val)) { + } else if (isinf(val)) { if (copysign(1., val) == 1.) strcpy(buf, "inf"); else diff --git a/Python/pytime.c b/Python/pytime.c index 560aea33f201a0..cd76970718622f 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -375,7 +375,7 @@ pytime_object_to_denominator(PyObject *obj, time_t *sec, long *numerator, if (PyFloat_Check(obj)) { double d = PyFloat_AsDouble(obj); - if (Py_IS_NAN(d)) { + if (isnan(d)) { *numerator = 0; PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; @@ -403,7 +403,7 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) volatile double d; d = PyFloat_AsDouble(obj); - if (Py_IS_NAN(d)) { + if (isnan(d)) { PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; } @@ -590,7 +590,7 @@ pytime_from_object(PyTime_t *tp, PyObject *obj, _PyTime_round_t round, if (PyFloat_Check(obj)) { double d; d = PyFloat_AsDouble(obj); - if (Py_IS_NAN(d)) { + if (isnan(d)) { PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; } From 7ca74a760a5d3cdf48159f003d4db7c2778e9261 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 29 May 2024 11:37:04 +0200 Subject: [PATCH 254/903] gh-119661: Add _Py_SINGLETON() include in Argumenet Clinic (#119712) When the _Py_SINGLETON() is used, Argument Clinic now adds an explicit "pycore_runtime.h" include to get the macro. Previously, the macro may or may not be included indirectly by another include. --- Modules/_ctypes/clinic/_ctypes.c.h | 5 ++++- Modules/_io/clinic/bufferedio.c.h | 4 ++-- Modules/_io/clinic/iobase.c.h | 5 ++++- Modules/_io/clinic/textio.c.h | 4 ++-- Modules/clinic/_curses_panel.c.h | 5 ++++- Modules/clinic/_dbmmodule.c.h | 5 ++++- Modules/clinic/_elementtree.c.h | 4 ++-- Modules/clinic/_gdbmmodule.c.h | 5 ++++- Modules/clinic/_pickle.c.h | 4 ++-- Modules/clinic/arraymodule.c.h | 5 ++++- Modules/clinic/pyexpat.c.h | 4 ++-- Tools/clinic/libclinic/parse_args.py | 12 +++++++++--- 12 files changed, 43 insertions(+), 19 deletions(-) diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h index 98a84cc14f4386..e1d5a17cbe7d68 100644 --- a/Modules/_ctypes/clinic/_ctypes.c.h +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -607,4 +610,4 @@ Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py } return Simple_from_outparm_impl(self, cls); } -/*[clinic end generated code: output=9c6539a3559e6088 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a90886be2a294ee6 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 64eddcd314a803..708bef638887e2 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() @@ -1245,4 +1245,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=4249187a725a3b3e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8eead000083dc5fa input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index bae80a265fab07..a35cac7dc0b8d7 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -438,4 +441,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=e7326fbefc52bfba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dab5e9323d191e32 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index f04ee729abc9ed..669e2aa637ebbf 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() @@ -1292,4 +1292,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=93a5a91a22100a28 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=04cb7c67791a9ec1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 457f71370afda9..c8788c461f745c 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, @@ -418,4 +421,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=7bac14e9a1194c87 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=18dc5571174c7189 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index d06271e18a49b2..4379b433db3738 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_dbm_dbm_close__doc__, @@ -218,4 +221,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=743ce0cea116747e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f7d9a87d80a64278 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 10b2dd1c15f7fd..1a5a820d1f00b5 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -1236,4 +1236,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=aed9f53eeb0404e0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bd28eba33d9c1f25 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 626e4678809d4f..bbf4365114c0aa 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_gdbm_gdbm_get__doc__, @@ -340,4 +343,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=6b4c19905ac9967d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=07bdeb4a8ecb328e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index 5a6ae7be6b6ea7..693c7d59e9d7a6 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -1077,4 +1077,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=bd63c85a8737b0aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c7dd60d20ee4895f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 8427f92de0d2e3..2ed7eaa6abf7af 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() @@ -688,4 +691,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=9a3276ffd499c796 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ecd63acd7924c223 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index e23c910d0ac661..682d8481a2a2f4 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -545,4 +545,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=51874bfaf4992ba2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9f1e9a7192d29976 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index 7ac88bd0458b82..0f67901dd8609a 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -38,6 +38,8 @@ def declare_parser( p for p in f.parameters.values() if not p.is_positional_only() and not p.is_vararg() ]) + + condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)' if limited_capi: declarations = """ #define KWTUPLE NULL @@ -50,6 +52,9 @@ def declare_parser( # define KWTUPLE NULL #endif """ + + codegen.add_include('pycore_runtime.h', '_Py_SINGLETON()', + condition=condition) else: # XXX Why do we not statically allocate the tuple # for non-builtin modules? @@ -73,9 +78,10 @@ def declare_parser( #endif // !Py_BUILD_CORE """ % num_keywords - condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)' - codegen.add_include('pycore_gc.h', 'PyGC_Head', condition=condition) - codegen.add_include('pycore_runtime.h', '_Py_ID()', condition=condition) + codegen.add_include('pycore_gc.h', 'PyGC_Head', + condition=condition) + codegen.add_include('pycore_runtime.h', '_Py_ID()', + condition=condition) declarations += """ static const char * const _keywords[] = {{{keywords_c} NULL}}; From c1e9647107c854439a9864b6ec4f6784aeb94ed5 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 29 May 2024 10:47:56 +0100 Subject: [PATCH 255/903] gh-119689: generate stack effect metadata for pseudo instructions (#119691) --- Include/internal/pycore_opcode_metadata.h | 32 +++++++++++ Lib/test/test_generated_cases.py | 4 +- ...-05-28-22-49-56.gh-issue-119689.VwLFD5.rst | 1 + Python/bytecodes.c | 24 +++++--- Python/compile.c | 57 +++++-------------- Tools/cases_generator/analyzer.py | 4 +- .../cases_generator/interpreter_definition.md | 8 ++- .../opcode_metadata_generator.py | 9 ++- Tools/cases_generator/parsing.py | 24 ++++---- Tools/cases_generator/stack.py | 22 ++++--- 10 files changed, 112 insertions(+), 73 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-28-22-49-56.gh-issue-119689.VwLFD5.rst diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index f805be04985ef2..d3535800139a66 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -259,12 +259,16 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case IS_OP: return 2; + case JUMP: + return 0; case JUMP_BACKWARD: return 0; case JUMP_BACKWARD_NO_INTERRUPT: return 0; case JUMP_FORWARD: return 0; + case JUMP_NO_INTERRUPT: + return 0; case LIST_APPEND: return 2 + (oparg-1); case LIST_EXTEND: @@ -297,6 +301,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case LOAD_BUILD_CLASS: return 0; + case LOAD_CLOSURE: + return 0; case LOAD_COMMON_CONSTANT: return 0; case LOAD_CONST: @@ -347,6 +353,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case NOP: return 0; + case POP_BLOCK: + return 0; case POP_EXCEPT: return 1; case POP_JUMP_IF_FALSE: @@ -385,6 +393,12 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case SETUP_ANNOTATIONS: return 0; + case SETUP_CLEANUP: + return 0; + case SETUP_FINALLY: + return 0; + case SETUP_WITH: + return 0; case SET_ADD: return 2 + (oparg-1); case SET_FUNCTION_ATTRIBUTE: @@ -405,6 +419,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case STORE_FAST_LOAD_FAST: return 1; + case STORE_FAST_MAYBE_NULL: + return 1; case STORE_FAST_STORE_FAST: return 2; case STORE_GLOBAL: @@ -692,12 +708,16 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case IS_OP: return 1; + case JUMP: + return 0; case JUMP_BACKWARD: return 0; case JUMP_BACKWARD_NO_INTERRUPT: return 0; case JUMP_FORWARD: return 0; + case JUMP_NO_INTERRUPT: + return 0; case LIST_APPEND: return 1 + (oparg-1); case LIST_EXTEND: @@ -730,6 +750,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1 + (oparg & 1); case LOAD_BUILD_CLASS: return 1; + case LOAD_CLOSURE: + return 1; case LOAD_COMMON_CONSTANT: return 1; case LOAD_CONST: @@ -780,6 +802,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2; case NOP: return 0; + case POP_BLOCK: + return 0; case POP_EXCEPT: return 0; case POP_JUMP_IF_FALSE: @@ -818,6 +842,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2; case SETUP_ANNOTATIONS: return 0; + case SETUP_CLEANUP: + return 2; + case SETUP_FINALLY: + return 1; + case SETUP_WITH: + return 1; case SET_ADD: return 1 + (oparg-1); case SET_FUNCTION_ATTRIBUTE: @@ -838,6 +868,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case STORE_FAST_LOAD_FAST: return 1; + case STORE_FAST_MAYBE_NULL: + return 0; case STORE_FAST_STORE_FAST: return 0; case STORE_GLOBAL: diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index fb85222fdcce74..41eeb9c0705741 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -485,7 +485,7 @@ def test_unused_caches(self): def test_pseudo_instruction_no_flags(self): input = """ - pseudo(OP) = { + pseudo(OP, (in -- out1, out2)) = { OP1, }; @@ -504,7 +504,7 @@ def test_pseudo_instruction_no_flags(self): def test_pseudo_instruction_with_flags(self): input = """ - pseudo(OP, (HAS_ARG, HAS_JUMP)) = { + pseudo(OP, (in1, in2 --), (HAS_ARG, HAS_JUMP)) = { OP1, }; diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-28-22-49-56.gh-issue-119689.VwLFD5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-28-22-49-56.gh-issue-119689.VwLFD5.rst new file mode 100644 index 00000000000000..56be31326216eb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-28-22-49-56.gh-issue-119689.VwLFD5.rst @@ -0,0 +1 @@ +Generate stack effect metadata for pseudo instructions from bytecodes.c. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 274c5c22447e4c..9a8198515dea5e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -213,7 +213,7 @@ dummy_func( } } - pseudo(LOAD_CLOSURE) = { + pseudo(LOAD_CLOSURE, (-- unused)) = { LOAD_FAST, }; @@ -259,7 +259,7 @@ dummy_func( SETLOCAL(oparg, value); } - pseudo(STORE_FAST_MAYBE_NULL) = { + pseudo(STORE_FAST_MAYBE_NULL, (unused --)) = { STORE_FAST, }; @@ -2393,12 +2393,12 @@ dummy_func( #endif /* _Py_TIER2 */ } - pseudo(JUMP) = { + pseudo(JUMP, (--)) = { JUMP_FORWARD, JUMP_BACKWARD, }; - pseudo(JUMP_NO_INTERRUPT) = { + pseudo(JUMP_NO_INTERRUPT, (--)) = { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, }; @@ -2895,19 +2895,27 @@ dummy_func( ERROR_IF(res == NULL, error); } - pseudo(SETUP_FINALLY, (HAS_ARG)) = { + pseudo(SETUP_FINALLY, (-- unused), (HAS_ARG)) = { + /* If an exception is raised, restore the stack position + * and push one value before jumping to the handler. + */ NOP, }; - pseudo(SETUP_CLEANUP, (HAS_ARG)) = { + pseudo(SETUP_CLEANUP, (-- unused, unused), (HAS_ARG)) = { + /* As SETUP_FINALLY, but push lasti as well */ NOP, }; - pseudo(SETUP_WITH, (HAS_ARG)) = { + pseudo(SETUP_WITH, (-- unused), (HAS_ARG)) = { + /* If an exception is raised, restore the stack position to the + * position before the result of __(a)enter__ and push 2 values + * before jumping to the handler. + */ NOP, }; - pseudo(POP_BLOCK) = { + pseudo(POP_BLOCK, (--)) = { NOP, }; diff --git a/Python/compile.c b/Python/compile.c index e6efae33eb45e4..3a80577e0f2b2d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -703,51 +703,22 @@ compiler_set_qualname(struct compiler *c) static int stack_effect(int opcode, int oparg, int jump) { - if (0 <= opcode && opcode <= MAX_REAL_OPCODE) { - if (_PyOpcode_Deopt[opcode] != opcode) { - // Specialized instructions are not supported. - return PY_INVALID_STACK_EFFECT; - } - int popped = _PyOpcode_num_popped(opcode, oparg); - int pushed = _PyOpcode_num_pushed(opcode, oparg); - if (popped < 0 || pushed < 0) { - return PY_INVALID_STACK_EFFECT; - } - return pushed - popped; + if (opcode < 0) { + return PY_INVALID_STACK_EFFECT; } - - // Pseudo ops - switch (opcode) { - case POP_BLOCK: - case JUMP: - case JUMP_NO_INTERRUPT: - return 0; - - /* Exception handling pseudo-instructions */ - case SETUP_FINALLY: - /* 0 in the normal flow. - * Restore the stack position and push 1 value before jumping to - * the handler if an exception be raised. */ - return jump ? 1 : 0; - case SETUP_CLEANUP: - /* As SETUP_FINALLY, but pushes lasti as well */ - return jump ? 2 : 0; - case SETUP_WITH: - /* 0 in the normal flow. - * Restore the stack position to the position before the result - * of __(a)enter__ and push 2 values before jumping to the handler - * if an exception be raised. */ - return jump ? 1 : 0; - - case STORE_FAST_MAYBE_NULL: - return -1; - case LOAD_CLOSURE: - return 1; - default: - return PY_INVALID_STACK_EFFECT; + if ((opcode <= MAX_REAL_OPCODE) && (_PyOpcode_Deopt[opcode] != opcode)) { + // Specialized instructions are not supported. + return PY_INVALID_STACK_EFFECT; } - - return PY_INVALID_STACK_EFFECT; /* not reachable */ + int popped = _PyOpcode_num_popped(opcode, oparg); + int pushed = _PyOpcode_num_pushed(opcode, oparg); + if (popped < 0 || pushed < 0) { + return PY_INVALID_STACK_EFFECT; + } + if (IS_BLOCK_PUSH_OPCODE(opcode) && !jump) { + return 0; + } + return pushed - popped; } int diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index fdb635486b9531..e44bebd8f3c4a4 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -235,6 +235,7 @@ def is_super(self) -> bool: @dataclass class PseudoInstruction: name: str + stack: StackEffect targets: list[Instruction] flags: list[str] opcode: int = -1 @@ -295,7 +296,7 @@ def convert_stack_item(item: parser.StackEffect, replace_op_arg_1: str | None) - item.name, item.type, cond, (item.size or "1") ) -def analyze_stack(op: parser.InstDef, replace_op_arg_1: str | None = None) -> StackEffect: +def analyze_stack(op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | None = None) -> StackEffect: inputs: list[StackItem] = [ convert_stack_item(i, replace_op_arg_1) for i in op.inputs if isinstance(i, parser.StackEffect) ] @@ -706,6 +707,7 @@ def add_pseudo( ) -> None: pseudos[pseudo.name] = PseudoInstruction( pseudo.name, + analyze_stack(pseudo), [instructions[target] for target in pseudo.targets], pseudo.flags, ) diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index 889f58fc3e1a75..ba09931c541646 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -124,7 +124,13 @@ and a piece of C code describing its semantics:: "family" "(" NAME ")" = "{" NAME ("," NAME)+ [","] "}" ";" pseudo: - "pseudo" "(" NAME ")" = "{" NAME ("," NAME)+ [","] "}" ";" + "pseudo" "(" NAME "," stack_effect ["," "(" flags ")"]")" = "{" NAME ("," NAME)+ [","] "}" ";" + + flags: + flag ("|" flag)* + + flag: + HAS_ARG | HAS_DEOPT | etc.. ``` The following definitions may occur: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 04fecb235f18cd..2632eb89ce80cd 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -10,6 +10,7 @@ from analyzer import ( Analysis, Instruction, + PseudoInstruction, analyze_files, Skip, Uop, @@ -94,12 +95,18 @@ def emit_stack_effect_function( def generate_stack_effect_functions(analysis: Analysis, out: CWriter) -> None: popped_data: list[tuple[str, str]] = [] pushed_data: list[tuple[str, str]] = [] - for inst in analysis.instructions.values(): + def add(inst: Instruction | PseudoInstruction) -> None: stack = get_stack_effect(inst) popped = (-stack.base_offset).to_c() pushed = (stack.top_offset - stack.base_offset).to_c() popped_data.append((inst.name, popped)) pushed_data.append((inst.name, pushed)) + + for inst in analysis.instructions.values(): + add(inst) + for pseudo in analysis.pseudos.values(): + add(pseudo) + emit_stack_effect_function(out, "popped", sorted(popped_data)) emit_stack_effect_function(out, "pushed", sorted(pushed_data)) diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 0d54820e4e71fb..cc897ff2cbe9aa 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -138,6 +138,8 @@ class Family(Node): @dataclass class Pseudo(Node): name: str + inputs: list[InputEffect] + outputs: list[OutputEffect] flags: list[str] # instr flags to set on the pseudo instruction targets: list[str] # opcodes this can be replaced by @@ -409,16 +411,18 @@ def pseudo_def(self) -> Pseudo | None: if self.expect(lx.LPAREN): if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.COMMA): - flags = self.flags() - else: - flags = [] - if self.expect(lx.RPAREN): - if self.expect(lx.EQUALS): - if not self.expect(lx.LBRACE): - raise self.make_syntax_error("Expected {") - if members := self.members(): - if self.expect(lx.RBRACE) and self.expect(lx.SEMI): - return Pseudo(tkn.text, flags, members) + inp, outp = self.io_effect() + if self.expect(lx.COMMA): + flags = self.flags() + else: + flags = [] + if self.expect(lx.RPAREN): + if self.expect(lx.EQUALS): + if not self.expect(lx.LBRACE): + raise self.make_syntax_error("Expected {") + if members := self.members(): + if self.expect(lx.RBRACE) and self.expect(lx.SEMI): + return Pseudo(tkn.text, inp, outp, flags, members) return None def members(self) -> list[str] | None: diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 5aecac39aef5e2..7f07a6805b1cb6 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -1,7 +1,8 @@ import re -from analyzer import StackItem, Instruction, Uop +from analyzer import StackItem, StackEffect, Instruction, Uop, PseudoInstruction from dataclasses import dataclass from cwriter import CWriter +from typing import Iterator UNUSED = {"unused"} @@ -208,13 +209,20 @@ def as_comment(self) -> str: return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" -def get_stack_effect(inst: Instruction) -> Stack: +def get_stack_effect(inst: Instruction | PseudoInstruction) -> Stack: stack = Stack() - for uop in inst.parts: - if not isinstance(uop, Uop): - continue - for var in reversed(uop.stack.inputs): + def stacks(inst : Instruction | PseudoInstruction) -> Iterator[StackEffect]: + if isinstance(inst, Instruction): + for uop in inst.parts: + if isinstance(uop, Uop): + yield uop.stack + else: + assert isinstance(inst, PseudoInstruction) + yield inst.stack + + for s in stacks(inst): + for var in reversed(s.inputs): stack.pop(var) - for i, var in enumerate(uop.stack.outputs): + for var in s.outputs: stack.push(var) return stack From 18c1a8d3a81bf8d287a06f2985bbf65c9a9b9794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20G=C3=B6rgens?= Date: Wed, 29 May 2024 18:02:53 +0800 Subject: [PATCH 256/903] gh-97588: Align ctypes struct layout to GCC/MSVC (GH-97702) Structure layout, and especially bitfields, sometimes resulted in clearly wrong behaviour like overlapping fields. This fixes Co-authored-by: Gregory P. Smith Co-authored-by: Petr Viktorin --- Doc/library/ctypes.rst | 45 +- Doc/whatsnew/3.13.rst | 12 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/test_ctypes/test_bitfields.py | 321 ++- .../test_ctypes/test_generated_structs.py | 718 +++++++ Makefile.pre.in | 1 + ...2-10-01-09-56-27.gh-issue-97588.Gvg54o.rst | 2 + Modules/_ctypes/_ctypes_test.c | 75 +- Modules/_ctypes/_ctypes_test_generated.c.h | 1885 +++++++++++++++++ Modules/_ctypes/cfield.c | 341 +-- Modules/_ctypes/ctypes.h | 11 +- Modules/_ctypes/stgdict.c | 49 +- PCbuild/_ctypes_test.vcxproj | 3 +- PCbuild/_ctypes_test.vcxproj.filters | 5 +- 17 files changed, 3286 insertions(+), 188 deletions(-) create mode 100644 Lib/test/test_ctypes/test_generated_structs.py create mode 100644 Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst create mode 100644 Modules/_ctypes/_ctypes_test_generated.c.h diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 820535e3cba106..29b35af1c858ee 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -661,14 +661,18 @@ for debugging because they can provide useful information:: guaranteed by the library to work in the general case. Unions and structures with bit-fields should always be passed to functions by pointer. -Structure/union alignment and byte order -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, Structure and Union fields are aligned in the same way the C -compiler does it. It is possible to override this behavior by specifying a -:attr:`~Structure._pack_` class attribute in the subclass definition. -This must be set to a positive integer and specifies the maximum alignment for the fields. -This is what ``#pragma pack(n)`` also does in MSVC. +Structure/union layout, alignment and byte order +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, Structure and Union fields are laid out in the same way the C +compiler does it. It is possible to override this behavior entirely by specifying a +:attr:`~Structure._layout_` class attribute in the subclass definition; see +the attribute documentation for details. + +It is possible to specify the maximum alignment for the fields by setting +the :attr:`~Structure._pack_` class attribute to a positive integer. +This matches what ``#pragma pack(n)`` does in MSVC. + It is also possible to set a minimum alignment for how the subclass itself is packed in the same way ``#pragma align(n)`` works in MSVC. This can be achieved by specifying a ::attr:`~Structure._align_` class attribute @@ -2540,6 +2544,31 @@ fields, or any other data types containing pointer type fields. the structure when being packed or unpacked to/from memory. Setting this attribute to 0 is the same as not setting it at all. + .. attribute:: _layout_ + + An optional string naming the struct/union layout. It can currently + be set to: + + - ``"ms"``: the layout used by the Microsoft compiler (MSVC). + On GCC and Clang, this layout can be selected with + ``__attribute__((ms_struct))``. + - ``"gcc-sysv"``: the layout used by GCC with the System V or “SysV-like” + data model, as used on Linux and macOS. + With this layout, :attr:`~Structure._pack_` must be unset or zero. + + If not set explicitly, ``ctypes`` will use a default that + matches the platform conventions. This default may change in future + Python releases (for example, when a new platform gains official support, + or when a difference between similar platforms is found). + Currently the default will be: + + - On Windows: ``"ms"`` + - When :attr:`~Structure._pack_` is specified: ``"ms"`` + - Otherwise: ``"gcc-sysv"`` + + :attr:`!_layout_` must already be defined when + :attr:`~Structure._fields_` is assigned, otherwise it will have no effect. + .. attribute:: _anonymous_ An optional sequence that lists the names of unnamed (anonymous) fields. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 7edfdd4f8167a0..2b1b5fdb974b26 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -656,6 +656,18 @@ copy any user classes which define the :meth:`!__replace__` method. (Contributed by Serhiy Storchaka in :gh:`108751`.) +ctypes +------ + +* The layout of :ref:`bit fields ` in + :class:`~ctypes.Structure` and :class:`~ctypes.Union` was improved to better + match platform defaults (GCC/Clang or MSC). In particular, fields no longer + overlap. + (Contributed by Matthias Görgens in :gh:`97702`.) +* A :attr:`ctypes.Structure._layout_` class attribute can be set + to help match a non-default ABI. + (Contributed by Petr Viktorin in :gh:`97702`.) + dbm --- diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 56a2d6b0f4fc5d..a0f8fb71c1ff37 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -767,6 +767,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_layout_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 657eac6c0a0f6c..57d85020f14e05 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -256,6 +256,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_initializing) STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) + STRUCT_FOR_ID(_layout_) STRUCT_FOR_ID(_length_) STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index f4f9c730e514e0..e62ebd659d30e8 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -765,6 +765,7 @@ extern "C" { INIT_ID(_initializing), \ INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ + INIT_ID(_layout_), \ INIT_ID(_length_), \ INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 33da27a941f024..892f580e8a6846 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -609,6 +609,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_is_text_encoding); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(_layout_); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_length_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 0332544b5827e6..e6509e6bf89e1d 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -1,9 +1,10 @@ import os +import sys import unittest from ctypes import (CDLL, Structure, sizeof, POINTER, byref, alignment, LittleEndianStructure, BigEndianStructure, c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, - c_uint32, c_uint64, + c_uint8, c_uint16, c_uint32, c_uint64, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong) from test import support from test.support import import_helper @@ -33,27 +34,88 @@ class BITS(Structure): func.argtypes = POINTER(BITS), c_char +class BITS_msvc(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9), + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + + +try: + func_msvc = CDLL(_ctypes_test.__file__).unpack_bitfields_msvc +except AttributeError as err: + # The MSVC struct must be available on Windows; it's optional elsewhere + if support.MS_WINDOWS: + raise err + func_msvc = None +else: + func_msvc.argtypes = POINTER(BITS_msvc), c_char + + class C_Test(unittest.TestCase): def test_ints(self): for i in range(512): for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) - # bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior - @support.skip_if_sanitizer(ub=True) def test_shorts(self): b = BITS() name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. if func(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. self.skipTest("Compiler does not support signed short bitfields") for i in range(256): for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) + + @unittest.skipUnless(func_msvc, "need MSVC or __attribute__((ms_struct))") + def test_shorts_msvc_mode(self): + b = BITS_msvc() + name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. + if func_msvc(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. + self.skipTest("Compiler does not support signed short bitfields") + for i in range(256): + for name in "MNOPQRS": + with self.subTest(i=i, name=name): + b = BITS_msvc() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func_msvc(byref(b), name.encode('ascii'))) signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) @@ -87,35 +149,41 @@ class X(Structure): def test_signed(self): for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)*2) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("dummy", c_typ), + ("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)*2) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) def test_unsigned(self): for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) def fail_fields(self, *fields): return self.get_except(type(Structure), "X", (), @@ -149,22 +217,28 @@ def test_c_wchar(self): def test_single_bitfield_size(self): for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + result = self.fail_fields(("a", c_typ, -1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) - result = self.fail_fields(("a", c_typ, 0)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + result = self.fail_fields(("a", c_typ, 0)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) - class X(Structure): - _fields_ = [("a", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) + class X(Structure): + _fields_ = [("a", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)) - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - self.assertEqual(sizeof(X), sizeof(c_typ)) + class X(Structure): + _fields_ = [("a", c_typ, sizeof(c_typ)*8)] + self.assertEqual(sizeof(X), sizeof(c_typ)) - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) def test_multi_bitfields_size(self): class X(Structure): @@ -236,6 +310,161 @@ class X(Structure): else: self.assertEqual(sizeof(X), sizeof(c_int) * 2) + def test_mixed_5(self): + class X(Structure): + _fields_ = [ + ('A', c_uint, 1), + ('B', c_ushort, 16)] + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + def test_mixed_6(self): + class X(Structure): + _fields_ = [ + ('A', c_ulonglong, 1), + ('B', c_uint, 32)] + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_7(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + self.assertEqual(16, sizeof(X)) + + def test_mixed_8(self): + class Foo(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1), + ] + + class Bar(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1), + ] + self.assertEqual(sizeof(Foo), sizeof(Bar)) + + def test_mixed_9(self): + class X(Structure): + _fields_ = [ + ("A", c_uint8), + ("B", c_uint32, 1), + ] + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_10(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32, 1), + ("B", c_uint64, 1), + ] + if sys.platform == 'win32': + self.assertEqual(8, alignment(X)) + self.assertEqual(16, sizeof(X)) + else: + self.assertEqual(8, alignment(X)) + self.assertEqual(8, sizeof(X)) + + def test_gh_95496(self): + for field_width in range(1, 33): + class TestStruct(Structure): + _fields_ = [ + ("Field1", c_uint32, field_width), + ("Field2", c_uint8, 8) + ] + + cmd = TestStruct() + cmd.Field2 = 1 + self.assertEqual(1, cmd.Field2) + + def test_gh_84039(self): + class Bad(Structure): + _pack_ = 1 + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + + class GoodA(Structure): + _pack_ = 1 + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ] + + + class Good(Structure): + _pack_ = 1 + _fields_ = [ + ("a", GoodA), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + self.assertEqual(3, sizeof(Bad)) + self.assertEqual(3, sizeof(Good)) + + def test_gh_73939(self): + class MyStructure(Structure): + _pack_ = 1 + _fields_ = [ + ("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2) + ] + self.assertEqual(8, sizeof(MyStructure)) + + def test_gh_86098(self): + class X(Structure): + _fields_ = [ + ("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16) + ] + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message class X(Structure): diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py new file mode 100644 index 00000000000000..f93371c067e8bd --- /dev/null +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -0,0 +1,718 @@ +"""Test CTypes structs, unions, bitfields against C equivalents. + +The types here are auto-converted to C source at +`Modules/_ctypes/_ctypes_test_generated.c.h`, which is compiled into +_ctypes_test. + +Run this module to regenerate the files: + +./python Lib/test/test_ctypes/test_generated_structs.py > Modules/_ctypes/_ctypes_test_generated.c.h +""" + +import unittest +from test.support import import_helper +import re +from dataclasses import dataclass +from functools import cached_property +import sys + +import ctypes +from ctypes import Structure, Union, _SimpleCData +from ctypes import sizeof, alignment, pointer, string_at +_ctypes_test = import_helper.import_module("_ctypes_test") + + +# ctypes erases the difference between `c_int` and e.g.`c_int16`. +# To keep it, we'll use custom subclasses with the C name stashed in `_c_name`: +class c_bool(ctypes.c_bool): + _c_name = '_Bool' + +# To do it for all the other types, use some metaprogramming: +for c_name, ctypes_name in { + 'signed char': 'c_byte', + 'short': 'c_short', + 'int': 'c_int', + 'long': 'c_long', + 'long long': 'c_longlong', + 'unsigned char': 'c_ubyte', + 'unsigned short': 'c_ushort', + 'unsigned int': 'c_uint', + 'unsigned long': 'c_ulong', + 'unsigned long long': 'c_ulonglong', + **{f'{u}int{n}_t': f'c_{u}int{n}' + for u in ('', 'u') + for n in (8, 16, 32, 64)} +}.items(): + ctype = getattr(ctypes, ctypes_name) + newtype = type(ctypes_name, (ctype,), {'_c_name': c_name}) + globals()[ctypes_name] = newtype + + +# Register structs and unions to test + +TESTCASES = {} +def register(name=None, set_name=False): + def decorator(cls, name=name): + if name is None: + name = cls.__name__ + assert name.isascii() # will be used in _PyUnicode_EqualToASCIIString + assert name.isidentifier() # will be used as a C identifier + assert name not in TESTCASES + TESTCASES[name] = cls + if set_name: + cls.__name__ = name + return cls + return decorator + +@register() +class SingleInt(Structure): + _fields_ = [('a', c_int)] + +@register() +class SingleInt_Union(Union): + _fields_ = [('a', c_int)] + + +@register() +class SingleU32(Structure): + _fields_ = [('a', c_uint32)] + + +@register() +class SimpleStruct(Structure): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class SimpleUnion(Union): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class ManyTypes(Structure): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class ManyTypesU(Union): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class Nested(Structure): + _fields_ = [ + ('a', SimpleStruct), ('b', SimpleUnion), ('anon', SimpleStruct), + ] + _anonymous_ = ['anon'] + + +@register() +class Packed1(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 1 + + +@register() +class Packed2(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 2 + + +@register() +class Packed3(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 4 + + +@register() +class Packed4(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 8 + +@register() +class X86_32EdgeCase(Structure): + # On a Pentium, long long (int64) is 32-bit aligned, + # so these are packed tightly. + _fields_ = [('a', c_int32), ('b', c_int64), ('c', c_int32)] + +@register() +class MSBitFieldExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('a', c_uint, 4), + ('b', c_uint, 5), + ('c', c_uint, 7)] + +@register() +class MSStraddlingExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('first', c_uint, 9), + ('second', c_uint, 7), + ('may_straddle', c_uint, 30), + ('last', c_uint, 18)] + +@register() +class IntBits(Structure): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits(Structure): + _fields_ = [*IntBits._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class IntBits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [*IntBits_MSVC._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +# Skipped for now -- we don't always match the alignment +#@register() +class IntBits_Union(Union): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +# Skipped for now -- we don't always match the alignment +#@register() +class BitsUnion(Union): + _fields_ = [*IntBits_Union._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class I64Bits(Structure): + _fields_ = [("a", c_int64, 1), + ("b", c_int64, 62), + ("c", c_int64, 1)] + +@register() +class U64Bits(Structure): + _fields_ = [("a", c_uint64, 1), + ("b", c_uint64, 62), + ("c", c_uint64, 1)] + +for n in 8, 16, 32, 64: + for signedness in '', 'u': + ctype = globals()[f'c_{signedness}int{n}'] + + @register(f'Struct331_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, 3), + ("c", ctype, 1)] + + @register(f'Struct1x1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct1nx1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("full", ctype), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct3xx_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, n-2), + ("c", ctype, n-2)] + +@register() +class Mixed1(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 4)] + +@register() +class Mixed2(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int32, 32)] + +@register() +class Mixed3(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + +@register() +class Mixed4(Structure): + _fields_ = [("a", c_short, 4), + ("b", c_short, 4), + ("c", c_int, 24), + ("d", c_short, 4), + ("e", c_short, 4), + ("f", c_int, 24)] + +@register() +class Mixed5(Structure): + _fields_ = [('A', c_uint, 1), + ('B', c_ushort, 16)] + +@register() +class Mixed6(Structure): + _fields_ = [('A', c_ulonglong, 1), + ('B', c_uint, 32)] + +@register() +class Mixed7(Structure): + _fields_ = [("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + +@register() +class Mixed8_a(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed8_b(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed9(Structure): + _fields_ = [("A", c_uint8), + ("B", c_uint32, 1)] + +@register() +class Mixed10(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_95496(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_84039_bad(Structure): + _pack_ = 1 + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_84039_good_a(Structure): + _pack_ = 1 + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1)] + +@register() +class Example_gh_84039_good(Structure): + _pack_ = 1 + _fields_ = [("a", Example_gh_84039_good_a), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_73939(Structure): + _pack_ = 1 + _fields_ = [("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2)] + +@register() +class Example_gh_86098(Structure): + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class Example_gh_86098_pack(Structure): + _pack_ = 1 + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class AnonBitfields(Structure): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + _anonymous_ = ["_"] + _fields_ = [("_", X), ('y', c_byte)] + + +class GeneratedTest(unittest.TestCase): + def test_generated_data(self): + """Check that a ctypes struct/union matches its C equivalent. + + This compares with data from get_generated_test_data(), a list of: + - name (str) + - size (int) + - alignment (int) + - for each field, three snapshots of memory, as bytes: + - memory after the field is set to -1 + - memory after the field is set to 1 + - memory after the field is set to 0 + + or: + - None + - reason to skip the test (str) + + This does depend on the C compiler keeping padding bits zero. + Common compilers seem to do so. + """ + for name, cls in TESTCASES.items(): + with self.subTest(name=name): + expected = iter(_ctypes_test.get_generated_test_data(name)) + expected_name = next(expected) + if expected_name is None: + self.skipTest(next(expected)) + self.assertEqual(name, expected_name) + self.assertEqual(sizeof(cls), next(expected)) + with self.subTest('alignment'): + self.assertEqual(alignment(cls), next(expected)) + obj = cls() + ptr = pointer(obj) + for field in iterfields(cls): + for value in -1, 1, 0: + with self.subTest(field=field.full_name, value=value): + field.set_to(obj, value) + py_mem = string_at(ptr, sizeof(obj)) + c_mem = next(expected) + if py_mem != c_mem: + # Generate a helpful failure message + lines, requires = dump_ctype(cls) + m = "\n".join([str(field), 'in:', *lines]) + self.assertEqual(py_mem.hex(), c_mem.hex(), m) + + +# The rest of this file is generating C code from a ctypes type. +# This is only meant for (and tested with) the known inputs in this file! + +def c_str_repr(string): + """Return a string as a C literal""" + return '"' + re.sub('([\"\'\\\\\n])', r'\\\1', string) + '"' + +def dump_simple_ctype(tp, variable_name='', semi=''): + """Get C type name or declaration of a scalar type + + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + length = getattr(tp, '_length_', None) + if length is not None: + return f'{dump_simple_ctype(tp._type_, variable_name)}[{length}]{semi}' + assert not issubclass(tp, (Structure, Union)) + return f'{tp._c_name}{maybe_space(variable_name)}{semi}' + + +def dump_ctype(tp, struct_or_union_tag='', variable_name='', semi=''): + """Get C type name or declaration of a ctype + + struct_or_union_tag: name of the struct or union + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + requires = set() + if issubclass(tp, (Structure, Union)): + attributes = [] + pushes = [] + pops = [] + pack = getattr(tp, '_pack_', None) + if pack is not None: + pushes.append(f'#pragma pack(push, {pack})') + pops.append(f'#pragma pack(pop)') + layout = getattr(tp, '_layout_', None) + if layout == 'ms' or pack: + # The 'ms_struct' attribute only works on x86 and PowerPC + requires.add( + 'defined(MS_WIN32) || (' + '(defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (' + 'defined(__GNUC__) || defined(__clang__)))' + ) + attributes.append('ms_struct') + if attributes: + a = f' GCC_ATTR({", ".join(attributes)})' + else: + a = '' + lines = [f'{struct_or_union(tp)}{a}{maybe_space(struct_or_union_tag)} ' +'{'] + for fielddesc in tp._fields_: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + if f_name in getattr(tp, '_anonymous_', ()): + f_name = '' + if f_bits is None: + subsemi = ';' + else: + if f_tp not in (c_int, c_uint): + # XLC can reportedly only handle int & unsigned int + # bitfields (the only types required by C spec) + requires.add('!defined(__xlc__)') + subsemi = f' :{f_bits};' + sub_lines, sub_requires = dump_ctype( + f_tp, variable_name=f_name, semi=subsemi) + requires.update(sub_requires) + for line in sub_lines: + lines.append(' ' + line) + lines.append(f'}}{maybe_space(variable_name)}{semi}') + return [*pushes, *lines, *reversed(pops)], requires + else: + return [dump_simple_ctype(tp, variable_name, semi)], requires + +def struct_or_union(cls): + if issubclass(cls, Structure): + return 'struct' + if issubclass(cls, Union): + return 'union' + raise TypeError(cls) + +def maybe_space(string): + if string: + return ' ' + string + return string + +def unpack_field_desc(f_name, f_tp, f_bits=None): + """Unpack a _fields_ entry into a (name, type, bits) triple""" + return f_name, f_tp, f_bits + +@dataclass +class FieldInfo: + """Information about a (possibly nested) struct/union field""" + name: str + tp: type + bits: int | None # number if this is a bit field + parent_type: type + parent: 'FieldInfo' #| None + + @cached_property + def attr_path(self): + """Attribute names to get at the value of this field""" + if self.name in getattr(self.parent_type, '_anonymous_', ()): + selfpath = () + else: + selfpath = (self.name,) + if self.parent: + return (*self.parent.attr_path, *selfpath) + else: + return selfpath + + @cached_property + def full_name(self): + """Attribute names to get at the value of this field""" + return '.'.join(self.attr_path) + + def set_to(self, obj, new): + """Set the field on a given Structure/Union instance""" + for attr_name in self.attr_path[:-1]: + obj = getattr(obj, attr_name) + setattr(obj, self.attr_path[-1], new) + + @cached_property + def root(self): + if self.parent is None: + return self + else: + return self.parent + + @cached_property + def descriptor(self): + return getattr(self.parent_type, self.name) + + def __repr__(self): + qname = f'{self.root.parent_type.__name__}.{self.full_name}' + try: + desc = self.descriptor + except AttributeError: + desc = '???' + return f'<{type(self).__name__} for {qname}: {desc}>' + +def iterfields(tp, parent=None): + """Get *leaf* fields of a structure or union, as FieldInfo""" + try: + fields = tp._fields_ + except AttributeError: + yield parent + else: + for fielddesc in fields: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + sub = FieldInfo(f_name, f_tp, f_bits, tp, parent) + yield from iterfields(f_tp, sub) + + +if __name__ == '__main__': + # Dump C source to stdout + def output(string): + print(re.compile(r'^ +$', re.MULTILINE).sub('', string).lstrip('\n')) + output(""" + /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + + + // Append VALUE to the result. + #define APPEND(ITEM) { \\ + PyObject *item = ITEM; \\ + if (!item) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + int rv = PyList_Append(result, item); \\ + Py_DECREF(item); \\ + if (rv < 0) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \\ + TYPE v = VAL; \\ + TARGET = v; \\ + APPEND(PyBytes_FromStringAndSize( \\ + (char*)&value, sizeof(value))); \\ + } + + // Set a field to -1, 1 and 0; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \\ + SET_AND_APPEND(TYPE, TARGET, -1) \\ + SET_AND_APPEND(TYPE, TARGET, 1) \\ + SET_AND_APPEND(TYPE, TARGET, 0) \\ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + """) + for name, cls in TESTCASES.items(): + output(""" + if (PyUnicode_CompareWithASCIIString(name, %s) == 0) { + """ % c_str_repr(name)) + lines, requires = dump_ctype(cls, struct_or_union_tag=name, semi=';') + if requires: + output(f""" + #if {" && ".join(f'({r})' for r in sorted(requires))} + """) + for line in lines: + output(' ' + line) + typename = f'{struct_or_union(cls)} {name}' + output(f""" + {typename} value = {{0}}; + APPEND(PyUnicode_FromString({c_str_repr(name)})); + APPEND(PyLong_FromLong(sizeof({typename}))); + APPEND(PyLong_FromLong(_Alignof({typename}))); + """.rstrip()) + for field in iterfields(cls): + f_tp = dump_simple_ctype(field.tp) + output(f"""\ + TEST_FIELD({f_tp}, value.{field.full_name}); + """.rstrip()) + if requires: + output(f""" + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + """) + output(""" + return result; + } + """) + + output(""" + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + """) diff --git a/Makefile.pre.in b/Makefile.pre.in index b259537eae6ecf..08918405403127 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3106,6 +3106,7 @@ MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/blake2module.h $(srcdir)/Modules/hashlib.h MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h +MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ diff --git a/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst b/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst new file mode 100644 index 00000000000000..0bb0f5bcd501ef --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst @@ -0,0 +1,2 @@ +Fix creating bitfields in :mod:`ctypes` structures and unions. Fields +no longer overlap. diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index f46f6362ddd03b..e9ff8108efaa2f 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -22,6 +22,8 @@ #define EXPORT(x) Py_EXPORTED_SYMBOL x +#include "_ctypes_test_generated.c.h" + /* some functions handy for testing */ EXPORT(int) @@ -343,6 +345,31 @@ _testfunc_bitfield_by_reference2(Test7 *in) { return result; } +typedef struct{ + uint16_t A ; + uint16_t B : 9; + uint16_t C : 1; + uint16_t D : 1; + uint16_t E : 1; + uint16_t F : 1; + uint16_t G : 3; + uint32_t H : 10; + uint32_t I : 20; + uint32_t J : 2; +} Test9; + +EXPORT(long) +_testfunc_bitfield_by_reference3(Test9 *in, long pos) { + long data[] = {in->A , in->B , in->C , in->D , in->E , in->F , in->G , in->H , in->I , in->J}; + long data_length = (long) (sizeof(data)/sizeof(data[0])); + if(pos < 0) + return -1; + if(pos >= data_length) + return -1; + + return data[pos]; +} + typedef union { signed int A: 1, B:2, C:3, D:2; } Test8; @@ -704,7 +731,7 @@ struct BITS { */ #ifndef __xlc__ #define SIGNED_SHORT_BITFIELDS - short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; + signed short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; #endif }; @@ -734,12 +761,58 @@ EXPORT(int) unpack_bitfields(struct BITS *bits, char name) return 999; } +#if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) +struct +#ifndef MS_WIN32 +__attribute__ ((ms_struct)) +#endif +BITS_msvc +{ + signed int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9; +/* + * The test case needs/uses "signed short" bitfields, but the + * IBM XLC compiler does not support this + */ +#ifndef __xlc__ +#define SIGNED_SHORT_BITFIELDS + signed short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; +#endif +}; + +EXPORT(int) unpack_bitfields_msvc(struct BITS_msvc *bits, char name) +{ + switch (name) { + case 'A': return bits->A; + case 'B': return bits->B; + case 'C': return bits->C; + case 'D': return bits->D; + case 'E': return bits->E; + case 'F': return bits->F; + case 'G': return bits->G; + case 'H': return bits->H; + case 'I': return bits->I; + +#ifdef SIGNED_SHORT_BITFIELDS + case 'M': return bits->M; + case 'N': return bits->N; + case 'O': return bits->O; + case 'P': return bits->P; + case 'Q': return bits->Q; + case 'R': return bits->R; + case 'S': return bits->S; +#endif + } + return 999; +} +#endif + static PyMethodDef module_methods[] = { /* {"get_last_tf_arg_s", get_last_tf_arg_s, METH_NOARGS}, {"get_last_tf_arg_u", get_last_tf_arg_u, METH_NOARGS}, */ {"func_si", py_func_si, METH_VARARGS}, {"func", py_func, METH_NOARGS}, + {"get_generated_test_data", get_generated_test_data, METH_O}, { NULL, NULL, 0, NULL}, }; diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h new file mode 100644 index 00000000000000..46a3e4b01e2259 --- /dev/null +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -0,0 +1,1885 @@ + /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + + + // Append VALUE to the result. + #define APPEND(ITEM) { \ + PyObject *item = ITEM; \ + if (!item) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + int rv = PyList_Append(result, item); \ + Py_DECREF(item); \ + if (rv < 0) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \ + TYPE v = VAL; \ + TARGET = v; \ + APPEND(PyBytes_FromStringAndSize( \ + (char*)&value, sizeof(value))); \ + } + + // Set a field to -1, 1 and 0; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \ + SET_AND_APPEND(TYPE, TARGET, -1) \ + SET_AND_APPEND(TYPE, TARGET, 1) \ + SET_AND_APPEND(TYPE, TARGET, 0) \ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleInt") == 0) { + + struct SingleInt { + int a; + }; + struct SingleInt value = {0}; + APPEND(PyUnicode_FromString("SingleInt")); + APPEND(PyLong_FromLong(sizeof(struct SingleInt))); + APPEND(PyLong_FromLong(_Alignof(struct SingleInt))); + TEST_FIELD(int, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleInt_Union") == 0) { + + union SingleInt_Union { + int a; + }; + union SingleInt_Union value = {0}; + APPEND(PyUnicode_FromString("SingleInt_Union")); + APPEND(PyLong_FromLong(sizeof(union SingleInt_Union))); + APPEND(PyLong_FromLong(_Alignof(union SingleInt_Union))); + TEST_FIELD(int, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleU32") == 0) { + + struct SingleU32 { + uint32_t a; + }; + struct SingleU32 value = {0}; + APPEND(PyUnicode_FromString("SingleU32")); + APPEND(PyLong_FromLong(sizeof(struct SingleU32))); + APPEND(PyLong_FromLong(_Alignof(struct SingleU32))); + TEST_FIELD(uint32_t, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SimpleStruct") == 0) { + + struct SimpleStruct { + int32_t x; + int8_t y; + uint16_t z; + }; + struct SimpleStruct value = {0}; + APPEND(PyUnicode_FromString("SimpleStruct")); + APPEND(PyLong_FromLong(sizeof(struct SimpleStruct))); + APPEND(PyLong_FromLong(_Alignof(struct SimpleStruct))); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SimpleUnion") == 0) { + + union SimpleUnion { + int32_t x; + int8_t y; + uint16_t z; + }; + union SimpleUnion value = {0}; + APPEND(PyUnicode_FromString("SimpleUnion")); + APPEND(PyLong_FromLong(sizeof(union SimpleUnion))); + APPEND(PyLong_FromLong(_Alignof(union SimpleUnion))); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "ManyTypes") == 0) { + + struct ManyTypes { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + }; + struct ManyTypes value = {0}; + APPEND(PyUnicode_FromString("ManyTypes")); + APPEND(PyLong_FromLong(sizeof(struct ManyTypes))); + APPEND(PyLong_FromLong(_Alignof(struct ManyTypes))); + TEST_FIELD(int8_t, value.i8); + TEST_FIELD(uint8_t, value.u8); + TEST_FIELD(int16_t, value.i16); + TEST_FIELD(uint16_t, value.u16); + TEST_FIELD(int32_t, value.i32); + TEST_FIELD(uint32_t, value.u32); + TEST_FIELD(int64_t, value.i64); + TEST_FIELD(uint64_t, value.u64); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "ManyTypesU") == 0) { + + union ManyTypesU { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + }; + union ManyTypesU value = {0}; + APPEND(PyUnicode_FromString("ManyTypesU")); + APPEND(PyLong_FromLong(sizeof(union ManyTypesU))); + APPEND(PyLong_FromLong(_Alignof(union ManyTypesU))); + TEST_FIELD(int8_t, value.i8); + TEST_FIELD(uint8_t, value.u8); + TEST_FIELD(int16_t, value.i16); + TEST_FIELD(uint16_t, value.u16); + TEST_FIELD(int32_t, value.i32); + TEST_FIELD(uint32_t, value.u32); + TEST_FIELD(int64_t, value.i64); + TEST_FIELD(uint64_t, value.u64); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Nested") == 0) { + + struct Nested { + struct { + int32_t x; + int8_t y; + uint16_t z; + } a; + union { + int32_t x; + int8_t y; + uint16_t z; + } b; + struct { + int32_t x; + int8_t y; + uint16_t z; + }; + }; + struct Nested value = {0}; + APPEND(PyUnicode_FromString("Nested")); + APPEND(PyLong_FromLong(sizeof(struct Nested))); + APPEND(PyLong_FromLong(_Alignof(struct Nested))); + TEST_FIELD(int32_t, value.a.x); + TEST_FIELD(int8_t, value.a.y); + TEST_FIELD(uint16_t, value.a.z); + TEST_FIELD(int32_t, value.b.x); + TEST_FIELD(int8_t, value.b.y); + TEST_FIELD(uint16_t, value.b.z); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed1") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Packed1 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed1 value = {0}; + APPEND(PyUnicode_FromString("Packed1")); + APPEND(PyLong_FromLong(sizeof(struct Packed1))); + APPEND(PyLong_FromLong(_Alignof(struct Packed1))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed2") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 2) + struct GCC_ATTR(ms_struct) Packed2 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed2 value = {0}; + APPEND(PyUnicode_FromString("Packed2")); + APPEND(PyLong_FromLong(sizeof(struct Packed2))); + APPEND(PyLong_FromLong(_Alignof(struct Packed2))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed3") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 4) + struct GCC_ATTR(ms_struct) Packed3 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed3 value = {0}; + APPEND(PyUnicode_FromString("Packed3")); + APPEND(PyLong_FromLong(sizeof(struct Packed3))); + APPEND(PyLong_FromLong(_Alignof(struct Packed3))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed4") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 8) + struct GCC_ATTR(ms_struct) Packed4 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed4 value = {0}; + APPEND(PyUnicode_FromString("Packed4")); + APPEND(PyLong_FromLong(sizeof(struct Packed4))); + APPEND(PyLong_FromLong(_Alignof(struct Packed4))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "X86_32EdgeCase") == 0) { + + struct X86_32EdgeCase { + int32_t a; + int64_t b; + int32_t c; + }; + struct X86_32EdgeCase value = {0}; + APPEND(PyUnicode_FromString("X86_32EdgeCase")); + APPEND(PyLong_FromLong(sizeof(struct X86_32EdgeCase))); + APPEND(PyLong_FromLong(_Alignof(struct X86_32EdgeCase))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int32_t, value.c); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "MSBitFieldExample") == 0) { + + struct MSBitFieldExample { + unsigned int a :4; + unsigned int b :5; + unsigned int c :7; + }; + struct MSBitFieldExample value = {0}; + APPEND(PyUnicode_FromString("MSBitFieldExample")); + APPEND(PyLong_FromLong(sizeof(struct MSBitFieldExample))); + APPEND(PyLong_FromLong(_Alignof(struct MSBitFieldExample))); + TEST_FIELD(unsigned int, value.a); + TEST_FIELD(unsigned int, value.b); + TEST_FIELD(unsigned int, value.c); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "MSStraddlingExample") == 0) { + + struct MSStraddlingExample { + unsigned int first :9; + unsigned int second :7; + unsigned int may_straddle :30; + unsigned int last :18; + }; + struct MSStraddlingExample value = {0}; + APPEND(PyUnicode_FromString("MSStraddlingExample")); + APPEND(PyLong_FromLong(sizeof(struct MSStraddlingExample))); + APPEND(PyLong_FromLong(_Alignof(struct MSStraddlingExample))); + TEST_FIELD(unsigned int, value.first); + TEST_FIELD(unsigned int, value.second); + TEST_FIELD(unsigned int, value.may_straddle); + TEST_FIELD(unsigned int, value.last); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "IntBits") == 0) { + + struct IntBits { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + struct IntBits value = {0}; + APPEND(PyUnicode_FromString("IntBits")); + APPEND(PyLong_FromLong(sizeof(struct IntBits))); + APPEND(PyLong_FromLong(_Alignof(struct IntBits))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Bits") == 0) { + + #if (!defined(__xlc__)) + + struct Bits { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + struct Bits value = {0}; + APPEND(PyUnicode_FromString("Bits")); + APPEND(PyLong_FromLong(sizeof(struct Bits))); + APPEND(PyLong_FromLong(_Alignof(struct Bits))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "IntBits_MSVC") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + struct GCC_ATTR(ms_struct) IntBits_MSVC { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + struct IntBits_MSVC value = {0}; + APPEND(PyUnicode_FromString("IntBits_MSVC")); + APPEND(PyLong_FromLong(sizeof(struct IntBits_MSVC))); + APPEND(PyLong_FromLong(_Alignof(struct IntBits_MSVC))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Bits_MSVC") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + struct GCC_ATTR(ms_struct) Bits_MSVC { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + struct Bits_MSVC value = {0}; + APPEND(PyUnicode_FromString("Bits_MSVC")); + APPEND(PyLong_FromLong(sizeof(struct Bits_MSVC))); + APPEND(PyLong_FromLong(_Alignof(struct Bits_MSVC))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "I64Bits") == 0) { + + #if (!defined(__xlc__)) + + struct I64Bits { + int64_t a :1; + int64_t b :62; + int64_t c :1; + }; + struct I64Bits value = {0}; + APPEND(PyUnicode_FromString("I64Bits")); + APPEND(PyLong_FromLong(sizeof(struct I64Bits))); + APPEND(PyLong_FromLong(_Alignof(struct I64Bits))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "U64Bits") == 0) { + + #if (!defined(__xlc__)) + + struct U64Bits { + uint64_t a :1; + uint64_t b :62; + uint64_t c :1; + }; + struct U64Bits value = {0}; + APPEND(PyUnicode_FromString("U64Bits")); + APPEND(PyLong_FromLong(sizeof(struct U64Bits))); + APPEND(PyLong_FromLong(_Alignof(struct U64Bits))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_8 { + int8_t a :3; + int8_t b :3; + int8_t c :1; + }; + struct Struct331_8 value = {0}; + APPEND(PyUnicode_FromString("Struct331_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_8 { + int8_t a :1; + int8_t b :6; + int8_t c :1; + }; + struct Struct1x1_8 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_8 { + int8_t a :1; + int8_t full; + int8_t b :6; + int8_t c :1; + }; + struct Struct1nx1_8 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.full); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_8 { + int8_t a :3; + int8_t b :6; + int8_t c :6; + }; + struct Struct3xx_8 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u8 { + uint8_t a :3; + uint8_t b :3; + uint8_t c :1; + }; + struct Struct331_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u8 { + uint8_t a :1; + uint8_t b :6; + uint8_t c :1; + }; + struct Struct1x1_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u8 { + uint8_t a :1; + uint8_t full; + uint8_t b :6; + uint8_t c :1; + }; + struct Struct1nx1_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.full); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u8 { + uint8_t a :3; + uint8_t b :6; + uint8_t c :6; + }; + struct Struct3xx_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_16 { + int16_t a :3; + int16_t b :3; + int16_t c :1; + }; + struct Struct331_16 value = {0}; + APPEND(PyUnicode_FromString("Struct331_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_16 { + int16_t a :1; + int16_t b :14; + int16_t c :1; + }; + struct Struct1x1_16 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_16 { + int16_t a :1; + int16_t full; + int16_t b :14; + int16_t c :1; + }; + struct Struct1nx1_16 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.full); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_16 { + int16_t a :3; + int16_t b :14; + int16_t c :14; + }; + struct Struct3xx_16 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u16 { + uint16_t a :3; + uint16_t b :3; + uint16_t c :1; + }; + struct Struct331_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u16 { + uint16_t a :1; + uint16_t b :14; + uint16_t c :1; + }; + struct Struct1x1_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u16 { + uint16_t a :1; + uint16_t full; + uint16_t b :14; + uint16_t c :1; + }; + struct Struct1nx1_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.full); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u16 { + uint16_t a :3; + uint16_t b :14; + uint16_t c :14; + }; + struct Struct3xx_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_32 { + int32_t a :3; + int32_t b :3; + int32_t c :1; + }; + struct Struct331_32 value = {0}; + APPEND(PyUnicode_FromString("Struct331_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_32 { + int32_t a :1; + int32_t b :30; + int32_t c :1; + }; + struct Struct1x1_32 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_32 { + int32_t a :1; + int32_t full; + int32_t b :30; + int32_t c :1; + }; + struct Struct1nx1_32 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.full); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_32 { + int32_t a :3; + int32_t b :30; + int32_t c :30; + }; + struct Struct3xx_32 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u32 { + uint32_t a :3; + uint32_t b :3; + uint32_t c :1; + }; + struct Struct331_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u32 { + uint32_t a :1; + uint32_t b :30; + uint32_t c :1; + }; + struct Struct1x1_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u32 { + uint32_t a :1; + uint32_t full; + uint32_t b :30; + uint32_t c :1; + }; + struct Struct1nx1_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.full); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u32 { + uint32_t a :3; + uint32_t b :30; + uint32_t c :30; + }; + struct Struct3xx_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_64 { + int64_t a :3; + int64_t b :3; + int64_t c :1; + }; + struct Struct331_64 value = {0}; + APPEND(PyUnicode_FromString("Struct331_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_64 { + int64_t a :1; + int64_t b :62; + int64_t c :1; + }; + struct Struct1x1_64 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_64 { + int64_t a :1; + int64_t full; + int64_t b :62; + int64_t c :1; + }; + struct Struct1nx1_64 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.full); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_64 { + int64_t a :3; + int64_t b :62; + int64_t c :62; + }; + struct Struct3xx_64 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u64 { + uint64_t a :3; + uint64_t b :3; + uint64_t c :1; + }; + struct Struct331_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u64 { + uint64_t a :1; + uint64_t b :62; + uint64_t c :1; + }; + struct Struct1x1_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u64 { + uint64_t a :1; + uint64_t full; + uint64_t b :62; + uint64_t c :1; + }; + struct Struct1nx1_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.full); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u64 { + uint64_t a :3; + uint64_t b :62; + uint64_t c :62; + }; + struct Struct3xx_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed1") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed1 { + signed char a :4; + int b :4; + }; + struct Mixed1 value = {0}; + APPEND(PyUnicode_FromString("Mixed1")); + APPEND(PyLong_FromLong(sizeof(struct Mixed1))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed1))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(int, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed2") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed2 { + signed char a :4; + int32_t b :32; + }; + struct Mixed2 value = {0}; + APPEND(PyUnicode_FromString("Mixed2")); + APPEND(PyLong_FromLong(sizeof(struct Mixed2))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed2))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(int32_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed3") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed3 { + signed char a :4; + unsigned char b :4; + }; + struct Mixed3 value = {0}; + APPEND(PyUnicode_FromString("Mixed3")); + APPEND(PyLong_FromLong(sizeof(struct Mixed3))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed3))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(unsigned char, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed4") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed4 { + short a :4; + short b :4; + int c :24; + short d :4; + short e :4; + int f :24; + }; + struct Mixed4 value = {0}; + APPEND(PyUnicode_FromString("Mixed4")); + APPEND(PyLong_FromLong(sizeof(struct Mixed4))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed4))); + TEST_FIELD(short, value.a); + TEST_FIELD(short, value.b); + TEST_FIELD(int, value.c); + TEST_FIELD(short, value.d); + TEST_FIELD(short, value.e); + TEST_FIELD(int, value.f); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed5") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed5 { + unsigned int A :1; + unsigned short B :16; + }; + struct Mixed5 value = {0}; + APPEND(PyUnicode_FromString("Mixed5")); + APPEND(PyLong_FromLong(sizeof(struct Mixed5))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed5))); + TEST_FIELD(unsigned int, value.A); + TEST_FIELD(unsigned short, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed6") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed6 { + unsigned long long A :1; + unsigned int B :32; + }; + struct Mixed6 value = {0}; + APPEND(PyUnicode_FromString("Mixed6")); + APPEND(PyLong_FromLong(sizeof(struct Mixed6))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed6))); + TEST_FIELD(unsigned long long, value.A); + TEST_FIELD(unsigned int, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed7") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed7 { + uint32_t A; + uint32_t B :20; + uint64_t C :24; + }; + struct Mixed7 value = {0}; + APPEND(PyUnicode_FromString("Mixed7")); + APPEND(PyLong_FromLong(sizeof(struct Mixed7))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed7))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(uint64_t, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed8_a") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed8_a { + uint32_t A; + uint32_t B :32; + unsigned long long C :1; + }; + struct Mixed8_a value = {0}; + APPEND(PyUnicode_FromString("Mixed8_a")); + APPEND(PyLong_FromLong(sizeof(struct Mixed8_a))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed8_a))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(unsigned long long, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed8_b") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed8_b { + uint32_t A; + uint32_t B; + unsigned long long C :1; + }; + struct Mixed8_b value = {0}; + APPEND(PyUnicode_FromString("Mixed8_b")); + APPEND(PyLong_FromLong(sizeof(struct Mixed8_b))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed8_b))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(unsigned long long, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed9") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed9 { + uint8_t A; + uint32_t B :1; + }; + struct Mixed9 value = {0}; + APPEND(PyUnicode_FromString("Mixed9")); + APPEND(PyLong_FromLong(sizeof(struct Mixed9))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed9))); + TEST_FIELD(uint8_t, value.A); + TEST_FIELD(uint32_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed10") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed10 { + uint32_t A :1; + uint64_t B :1; + }; + struct Mixed10 value = {0}; + APPEND(PyUnicode_FromString("Mixed10")); + APPEND(PyLong_FromLong(sizeof(struct Mixed10))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed10))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint64_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_95496") == 0) { + + #if (!defined(__xlc__)) + + struct Example_gh_95496 { + uint32_t A :1; + uint64_t B :1; + }; + struct Example_gh_95496 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_95496")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_95496))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_95496))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint64_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_bad") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_bad { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + uint16_t b0 :4; + uint16_t b1 :12; + }; + #pragma pack(pop) + struct Example_gh_84039_bad value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_bad")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_bad))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_bad))); + TEST_FIELD(uint8_t, value.a0); + TEST_FIELD(uint8_t, value.a1); + TEST_FIELD(uint8_t, value.a2); + TEST_FIELD(uint8_t, value.a3); + TEST_FIELD(uint8_t, value.a4); + TEST_FIELD(uint8_t, value.a5); + TEST_FIELD(uint8_t, value.a6); + TEST_FIELD(uint8_t, value.a7); + TEST_FIELD(uint16_t, value.b0); + TEST_FIELD(uint16_t, value.b1); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good_a") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_good_a { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + }; + #pragma pack(pop) + struct Example_gh_84039_good_a value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_good_a")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good_a))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_good_a))); + TEST_FIELD(uint8_t, value.a0); + TEST_FIELD(uint8_t, value.a1); + TEST_FIELD(uint8_t, value.a2); + TEST_FIELD(uint8_t, value.a3); + TEST_FIELD(uint8_t, value.a4); + TEST_FIELD(uint8_t, value.a5); + TEST_FIELD(uint8_t, value.a6); + TEST_FIELD(uint8_t, value.a7); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_good { + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + } a; + #pragma pack(pop) + uint16_t b0 :4; + uint16_t b1 :12; + }; + #pragma pack(pop) + struct Example_gh_84039_good value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_good")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_good))); + TEST_FIELD(uint8_t, value.a.a0); + TEST_FIELD(uint8_t, value.a.a1); + TEST_FIELD(uint8_t, value.a.a2); + TEST_FIELD(uint8_t, value.a.a3); + TEST_FIELD(uint8_t, value.a.a4); + TEST_FIELD(uint8_t, value.a.a5); + TEST_FIELD(uint8_t, value.a.a6); + TEST_FIELD(uint8_t, value.a.a7); + TEST_FIELD(uint16_t, value.b0); + TEST_FIELD(uint16_t, value.b1); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_73939") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_73939 { + uint16_t P; + uint16_t L :9; + uint16_t Pro :1; + uint16_t G :1; + uint16_t IB :1; + uint16_t IR :1; + uint16_t R :3; + uint32_t T :10; + uint32_t C :20; + uint32_t R2 :2; + }; + #pragma pack(pop) + struct Example_gh_73939 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_73939")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_73939))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_73939))); + TEST_FIELD(uint16_t, value.P); + TEST_FIELD(uint16_t, value.L); + TEST_FIELD(uint16_t, value.Pro); + TEST_FIELD(uint16_t, value.G); + TEST_FIELD(uint16_t, value.IB); + TEST_FIELD(uint16_t, value.IR); + TEST_FIELD(uint16_t, value.R); + TEST_FIELD(uint32_t, value.T); + TEST_FIELD(uint32_t, value.C); + TEST_FIELD(uint32_t, value.R2); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098") == 0) { + + #if (!defined(__xlc__)) + + struct Example_gh_86098 { + uint8_t a :8; + uint8_t b :8; + uint32_t c :16; + }; + struct Example_gh_86098 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_86098")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_86098))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098_pack") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_86098_pack { + uint8_t a :8; + uint8_t b :8; + uint32_t c :16; + }; + #pragma pack(pop) + struct Example_gh_86098_pack value = {0}; + APPEND(PyUnicode_FromString("Example_gh_86098_pack")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098_pack))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_86098_pack))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "AnonBitfields") == 0) { + + #if (!defined(__xlc__)) + + struct AnonBitfields { + struct { + signed char a :4; + unsigned char b :4; + }; + signed char y; + }; + struct AnonBitfields value = {0}; + APPEND(PyUnicode_FromString("AnonBitfields")); + APPEND(PyLong_FromLong(sizeof(struct AnonBitfields))); + APPEND(PyLong_FromLong(_Alignof(struct AnonBitfields))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(unsigned char, value.b); + TEST_FIELD(signed char, value.y); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 7c98b0f7e31a46..fa5213ca76d54f 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -31,40 +31,168 @@ static void pymem_destructor(PyObject *ptr) PyCField_Type */ -/* - * Expects the size, index and offset for the current field in *psize and - * *poffset, stores the total size so far in *psize, the offset for the next - * field in *poffset, the alignment requirements for the current field in - * *palign, and returns a field descriptor for this field. - */ -/* - * bitfields extension: - * bitsize != 0: this is a bit field. - * pbitofs points to the current bit offset, this will be updated. - * prev_desc points to the type of the previous bitfield, if any. - */ +static inline +Py_ssize_t round_down(Py_ssize_t numToRound, Py_ssize_t multiple) +{ + assert(numToRound >= 0); + assert(multiple >= 0); + if (multiple == 0) + return numToRound; + return (numToRound / multiple) * multiple; +} + +static inline +Py_ssize_t round_up(Py_ssize_t numToRound, Py_ssize_t multiple) +{ + assert(numToRound >= 0); + assert(multiple >= 0); + if (multiple == 0) + return numToRound; + return ((numToRound + multiple - 1) / multiple) * multiple; +} + +static inline +Py_ssize_t NUM_BITS(Py_ssize_t bitsize); +static inline +Py_ssize_t LOW_BIT(Py_ssize_t offset); +static inline +Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset); + +/* PyCField_FromDesc creates and returns a struct/union field descriptor. + +The function expects to be called repeatedly for all fields in a struct or +union. It uses helper functions PyCField_FromDesc_gcc and +PyCField_FromDesc_msvc to simulate the corresponding compilers. + +GCC mode places fields one after another, bit by bit. But "each bit field must +fit within a single object of its specified type" (GCC manual, section 15.8 +"Bit Field Packing"). When it doesn't, we insert a few bits of padding to +avoid that. + +MSVC mode works similar except for bitfield packing. Adjacent bit-fields are +packed into the same 1-, 2-, or 4-byte allocation unit if the integral types +are the same size and if the next bit-field fits into the current allocation +unit without crossing the boundary imposed by the common alignment requirements +of the bit-fields. + +See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields for details. + +We do not support zero length bitfields. In fact we use bitsize != 0 elsewhere +to indicate a bitfield. Here, non-bitfields need bitsize set to size*8. + +PyCField_FromDesc manages: +- *psize: the size of the structure / union so far. +- *poffset, *pbitofs: 8* (*poffset) + *pbitofs points to where the next field + would start. +- *palign: the alignment requirements of the last field we placed. +*/ + +static int +PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, + Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, + CFieldObject* self, StgInfo* info, + int is_bitfield + ) +{ + // We don't use poffset here, so clear it, if it has been set. + *pbitofs += *poffset * 8; + *poffset = 0; + + *palign = info->align; + + if (bitsize > 0) { + // Determine whether the bit field, if placed at the next free bit, + // fits within a single object of its specified type. + // That is: determine a "slot", sized & aligned for the specified type, + // which contains the bitfield's beginning: + Py_ssize_t slot_start_bit = round_down(*pbitofs, 8 * info->align); + Py_ssize_t slot_end_bit = slot_start_bit + 8 * info->size; + // And see if it also contains the bitfield's last bit: + Py_ssize_t field_end_bit = *pbitofs + bitsize; + if (field_end_bit > slot_end_bit) { + // It doesn't: add padding (bump up to the next alignment boundary) + *pbitofs = round_up(*pbitofs, 8*info->align); + } + } + assert(*poffset == 0); + + self->offset = round_down(*pbitofs, 8*info->align) / 8; + if(is_bitfield) { + Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; + self->size = BUILD_SIZE(bitsize, effective_bitsof); + assert(effective_bitsof <= info->size * 8); + } else { + self->size = info->size; + } + + *pbitofs += bitsize; + *psize = round_up(*pbitofs, 8) / 8; + + return 0; +} + +static int +PyCField_FromDesc_msvc( + Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, + Py_ssize_t *palign, int pack, + CFieldObject* self, StgInfo* info, + int is_bitfield + ) +{ + if (pack) { + *palign = Py_MIN(pack, info->align); + } else { + *palign = info->align; + } + + // *poffset points to end of current bitfield. + // *pbitofs is generally non-positive, + // and 8 * (*poffset) + *pbitofs points just behind + // the end of the last field we placed. + if (0 < *pbitofs + bitsize || 8 * info->size != *pfield_size) { + // Close the previous bitfield (if any). + // and start a new bitfield: + *poffset = round_up(*poffset, *palign); + + *poffset += info->size; + + *pfield_size = info->size * 8; + // Reminder: 8 * (*poffset) + *pbitofs points to where we would start a + // new field. Ie just behind where we placed the last field plus an + // allowance for alignment. + *pbitofs = - *pfield_size; + } + + assert(8 * info->size == *pfield_size); + + self->offset = *poffset - (*pfield_size) / 8; + if(is_bitfield) { + assert(0 <= (*pfield_size + *pbitofs)); + assert((*pfield_size + *pbitofs) < info->size * 8); + self->size = BUILD_SIZE(bitsize, *pfield_size + *pbitofs); + } else { + self->size = info->size; + } + assert(*pfield_size + *pbitofs <= info->size * 8); + + *pbitofs += bitsize; + *psize = *poffset; + + return 0; +} + PyObject * PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian) + Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, + int pack, int big_endian, LayoutMode layout_mode) { - CFieldObject *self; - PyObject *proto; - Py_ssize_t size, align; - SETFUNC setfunc = NULL; - GETFUNC getfunc = NULL; - int fieldtype; -#define NO_BITFIELD 0 -#define NEW_BITFIELD 1 -#define CONT_BITFIELD 2 -#define EXPAND_BITFIELD 3 - PyTypeObject *tp = st->PyCField_Type; - self = (CFieldObject *)tp->tp_alloc(tp, 0); - if (self == NULL) + CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); + if (self == NULL) { return NULL; - + } StgInfo *info; if (PyStgInfo_FromType(st, desc, &info) < 0) { Py_DECREF(self); @@ -77,44 +205,13 @@ PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, return NULL; } - if (bitsize /* this is a bitfield request */ - && *pfield_size /* we have a bitfield open */ -#ifdef MS_WIN32 - /* MSVC, GCC with -mms-bitfields */ - && info->size * 8 == *pfield_size -#else - /* GCC */ - && info->size * 8 <= *pfield_size -#endif - && (*pbitofs + bitsize) <= *pfield_size) { - /* continue bit field */ - fieldtype = CONT_BITFIELD; -#ifndef MS_WIN32 - } else if (bitsize /* this is a bitfield request */ - && *pfield_size /* we have a bitfield open */ - && info->size * 8 >= *pfield_size - && (*pbitofs + bitsize) <= info->size * 8) { - /* expand bit field */ - fieldtype = EXPAND_BITFIELD; -#endif - } else if (bitsize) { - /* start new bitfield */ - fieldtype = NEW_BITFIELD; - *pbitofs = 0; - *pfield_size = info->size * 8; - } else { - /* not a bit field */ - fieldtype = NO_BITFIELD; - *pbitofs = 0; - *pfield_size = 0; - } - - size = info->size; - proto = desc; + PyObject* proto = desc; /* Field descriptors for 'c_char * n' are be scpecial cased to return a Python string instead of an Array object instance... */ + SETFUNC setfunc = NULL; + GETFUNC getfunc = NULL; if (PyCArrayTypeObject_Check(st, proto)) { StgInfo *ainfo; if (PyStgInfo_FromType(st, proto, &ainfo) < 0) { @@ -153,61 +250,43 @@ PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, self->proto = Py_NewRef(proto); - switch (fieldtype) { - case NEW_BITFIELD: - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - *pbitofs = bitsize; - /* fall through */ - case NO_BITFIELD: - if (pack) - align = min(pack, info->align); - else - align = info->align; - if (align && *poffset % align) { - Py_ssize_t delta = align - (*poffset % align); - *psize += delta; - *poffset += delta; - } - - if (bitsize == 0) - self->size = size; - *psize += size; - - self->offset = *poffset; - *poffset += size; - - *palign = align; - break; - - case EXPAND_BITFIELD: - *poffset += info->size - *pfield_size/8; - *psize += info->size - *pfield_size/8; - - *pfield_size = info->size * 8; - - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - - self->offset = *poffset - size; /* poffset is already updated for the NEXT field */ - *pbitofs += bitsize; - break; - - case CONT_BITFIELD: - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - - self->offset = *poffset - size; /* poffset is already updated for the NEXT field */ - *pbitofs += bitsize; - break; + int is_bitfield = !!bitsize; + if(!is_bitfield) { + assert(info->size >= 0); + // assert: no overflow; + assert((unsigned long long int) info->size + < (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8); + bitsize = 8 * info->size; + // Caution: bitsize might still be 0 now. + } + assert(bitsize <= info->size * 8); + + int result; + if (layout_mode == LAYOUT_MODE_MS) { + result = PyCField_FromDesc_msvc( + pfield_size, bitsize, pbitofs, + psize, poffset, palign, + pack, + self, info, + is_bitfield + ); + } else { + assert(pack == 0); + result = PyCField_FromDesc_gcc( + bitsize, pbitofs, + psize, poffset, palign, + self, info, + is_bitfield + ); + } + if (result < 0) { + Py_DECREF(self); + return NULL; + } + assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); + if(big_endian && is_bitfield) { + self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); } - return (PyObject *)self; } @@ -298,8 +377,8 @@ static PyObject * PyCField_repr(CFieldObject *self) { PyObject *result; - Py_ssize_t bits = self->size >> 16; - Py_ssize_t size = self->size & 0xFFFF; + Py_ssize_t bits = NUM_BITS(self->size); + Py_ssize_t size = LOW_BIT(self->size); const char *name; name = ((PyTypeObject *)self->proto)->tp_name; @@ -396,8 +475,28 @@ get_ulonglong(PyObject *v, unsigned long long *p) */ /* how to decode the size field, for integer get/set functions */ -#define LOW_BIT(x) ((x) & 0xFFFF) -#define NUM_BITS(x) ((x) >> 16) +static inline +Py_ssize_t LOW_BIT(Py_ssize_t offset) { + return offset & 0xFFFF; +} +static inline +Py_ssize_t NUM_BITS(Py_ssize_t bitsize) { + return bitsize >> 16; +} + +static inline +Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset) { + assert(0 <= offset); + assert(offset <= 0xFFFF); + // We don't support zero length bitfields. + // And GET_BITFIELD uses NUM_BITS(size)==0, + // to figure out whether we are handling a bitfield. + assert(0 < bitsize); + Py_ssize_t result = (bitsize << 16) + offset; + assert(bitsize == NUM_BITS(result)); + assert(offset == LOW_BIT(result)); + return result; +} /* Doesn't work if NUM_BITS(size) == 0, but it never happens in SET() call. */ #define BIT_MASK(type, size) (((((type)1 << (NUM_BITS(size) - 1)) - 1) << 1) + 1) diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 423120f3460113..2d711dabab6c77 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -210,12 +210,17 @@ extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palig extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); +typedef enum { + LAYOUT_MODE_MS, + LAYOUT_MODE_GCC_SYSV, +} LayoutMode; extern PyObject * PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int is_big_endian); + Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, + Py_ssize_t *palign, + int pack, int is_big_endian, LayoutMode layout_mode); extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index ad82e4891c519a..52d8ec92380b30 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -243,7 +243,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t len, offset, size, align, i; Py_ssize_t union_size, total_align, aligned_size; Py_ssize_t field_size = 0; - int bitofs; + Py_ssize_t bitofs = 0; PyObject *tmp; int pack; int forced_alignment = 1; @@ -287,6 +287,38 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct pack = 0; } + #ifdef MS_WIN32 + LayoutMode layout_mode = LAYOUT_MODE_MS; + #else + LayoutMode layout_mode = (pack > 0) ? LAYOUT_MODE_MS : LAYOUT_MODE_GCC_SYSV; + #endif + + if (PyObject_GetOptionalAttr(type, &_Py_ID(_layout_), &tmp) < 0) { + return -1; + } + if (tmp) { + if (!PyUnicode_Check(tmp)) { + PyErr_SetString(PyExc_TypeError, + "_layout_ must be a string"); + return -1; + } + if (PyUnicode_CompareWithASCIIString(tmp, "ms") == 0) { + layout_mode = LAYOUT_MODE_MS; + } + else if (PyUnicode_CompareWithASCIIString(tmp, "gcc-sysv") == 0) { + layout_mode = LAYOUT_MODE_GCC_SYSV; + if (pack > 0) { + PyErr_SetString(PyExc_ValueError, + "_pack_ is not compatible with _layout_=\"gcc-sysv\""); + return -1; + } + } + else { + PyErr_Format(PyExc_ValueError, + "unknown _layout_ %R", tmp); + return -1; + } + } if (PyObject_GetOptionalAttr(type, &_Py_ID(_align_), &tmp) < 0) { return -1; } @@ -409,9 +441,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyObject *name = NULL, *desc = NULL; PyObject *pair = PySequence_GetItem(fields, i); PyObject *prop; - int bitsize = 0; + Py_ssize_t bitsize = 0; - if (!pair || !PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { + if (!pair || !PyArg_ParseTuple(pair, "UO|n", &name, &desc, &bitsize)) { PyErr_SetString(PyExc_TypeError, "'_fields_' must be a sequence of (name, C type) pairs"); Py_XDECREF(pair); @@ -465,8 +497,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return -1; } if (bitsize <= 0 || bitsize > info->size * 8) { - PyErr_SetString(PyExc_ValueError, - "number of bits invalid for bit field"); + PyErr_Format(PyExc_ValueError, + "number of bits invalid for bit field %R", + name); Py_DECREF(pair); return -1; } @@ -493,7 +526,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct prop = PyCField_FromDesc(st, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian); + pack, big_endian, layout_mode); if (prop == NULL) { Py_DECREF(pair); return -1; @@ -541,13 +574,15 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return -1; } } else /* union */ { + field_size = 0; size = 0; + bitofs = 0; offset = 0; align = 0; prop = PyCField_FromDesc(st, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian); + pack, big_endian, layout_mode); if (prop == NULL) { Py_DECREF(pair); return -1; diff --git a/PCbuild/_ctypes_test.vcxproj b/PCbuild/_ctypes_test.vcxproj index 97354739c09834..50d8575ad7bda3 100644 --- a/PCbuild/_ctypes_test.vcxproj +++ b/PCbuild/_ctypes_test.vcxproj @@ -94,6 +94,7 @@ + @@ -109,4 +110,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_ctypes_test.vcxproj.filters b/PCbuild/_ctypes_test.vcxproj.filters index 5174196c52e4d0..618cfb32115e99 100644 --- a/PCbuild/_ctypes_test.vcxproj.filters +++ b/PCbuild/_ctypes_test.vcxproj.filters @@ -15,6 +15,9 @@ Header Files + + Header Files + @@ -26,4 +29,4 @@ Resource Files - \ No newline at end of file + From 016a46ab572fc681e4a06760147b9ae311b21881 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 29 May 2024 11:44:04 +0100 Subject: [PATCH 257/903] gh-93554: add test for quickening of code in loops ending with conditional statement (#119485) --- Lib/test/test_dis.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 67a630e1346109..b1a1b77c53e8cb 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1207,6 +1207,36 @@ def test_loop_quicken(self): expected = dis_loop_test_quickened_code self.do_disassembly_compare(got, expected) + @cpython_only + @requires_specialization + def test_loop_with_conditional_at_end_is_quickened(self): + def for_loop_true(x): + for i in range(10): + if x: + pass + + for_loop_true(True) + self.assertIn('FOR_ITER_RANGE', + self.get_disassembly(for_loop_true, adaptive=True)) + + def for_loop_false(x): + for i in range(10): + if x: + pass + + for_loop_false(False) + self.assertIn('FOR_ITER_RANGE', + self.get_disassembly(for_loop_false, adaptive=True)) + + def while_loop(): + i = 0 + while i < 10: + i += 1 + + while_loop() + self.assertIn('COMPARE_OP_INT', + self.get_disassembly(while_loop, adaptive=True)) + @cpython_only def test_extended_arg_quick(self): got = self.get_disassembly(extended_arg_quick) From 0cdc5c8d5446e5eedd34e03d8e380bc237d1379b Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 29 May 2024 13:45:14 +0300 Subject: [PATCH 258/903] gh-119613: Soft deprecate Py_IS_NAN/INFINITY/FINITE (#119701) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Erlend E. Aasland Co-authored-by: Victor Stinner --- Doc/whatsnew/3.14.rst | 6 ++++++ Include/pymath.h | 3 +++ .../C API/2024-05-29-09-21-37.gh-issue-119613.J2xfrC.rst | 2 ++ 3 files changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2024-05-29-09-21-37.gh-issue-119613.J2xfrC.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 15216479cc6e5c..bc7fe64e68bb18 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -231,6 +231,12 @@ Porting to Python 3.14 Deprecated ---------- +* Macros :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY` + and :c:macro:`!Py_IS_FINITE` are :term:`soft deprecated`, + use instead :c:macro:`!isnan`, :c:macro:`!isinf` and + :c:macro:`!isfinite` available from :file:`math.h` + since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) + Removed ------- diff --git a/Include/pymath.h b/Include/pymath.h index 4c1e3d9984894b..d8f763f808d662 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -29,14 +29,17 @@ // Py_IS_NAN(X) // Return 1 if float or double arg is a NaN, else 0. +// Soft deprecated since Python 3.14, use isnan() instead. #define Py_IS_NAN(X) isnan(X) // Py_IS_INFINITY(X) // Return 1 if float or double arg is an infinity, else 0. +// Soft deprecated since Python 3.14, use isinf() instead. #define Py_IS_INFINITY(X) isinf(X) // Py_IS_FINITE(X) // Return 1 if float or double arg is neither infinite nor NAN, else 0. +// Soft deprecated since Python 3.14, use isfinite() instead. #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. diff --git a/Misc/NEWS.d/next/C API/2024-05-29-09-21-37.gh-issue-119613.J2xfrC.rst b/Misc/NEWS.d/next/C API/2024-05-29-09-21-37.gh-issue-119613.J2xfrC.rst new file mode 100644 index 00000000000000..196a4722a98c70 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-29-09-21-37.gh-issue-119613.J2xfrC.rst @@ -0,0 +1,2 @@ +Macros ``Py_IS_NAN``, ``Py_IS_INFINITY`` and ``Py_IS_FINITE`` +are :term:`soft deprecated`. From 055c739536ad63b55ad7cd0b91ccacc33064fe11 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Wed, 29 May 2024 05:13:18 -0700 Subject: [PATCH 259/903] CI: set correct working directory for Hypothesis cache (GH-119345) Set cwd for Hypothesis database --- .github/workflows/build.yml | 4 ++-- Lib/test/support/hypothesis_helper.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d14d17a5e088b6..7e63737b90b72a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -388,7 +388,7 @@ jobs: id: cache-hypothesis-database uses: actions/cache@v4 with: - path: ./hypothesis + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} restore-keys: | - hypothesis-database- @@ -416,7 +416,7 @@ jobs: if: always() with: name: hypothesis-example-db - path: .hypothesis/examples/ + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ build_asan: diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py index db93eea5e912e0..40f58a2f59c6c3 100644 --- a/Lib/test/support/hypothesis_helper.py +++ b/Lib/test/support/hypothesis_helper.py @@ -5,6 +5,13 @@ except ImportError: from . import _hypothesis_stubs as hypothesis else: + # Regrtest changes to use a tempdir as the working directory, so we have + # to tell Hypothesis to use the original in order to persist the database. + from .os_helper import SAVEDCWD + from hypothesis.configuration import set_hypothesis_home_dir + + set_hypothesis_home_dir(os.path.join(SAVEDCWD, ".hypothesis")) + # When using the real Hypothesis, we'll configure it to ignore occasional # slow tests (avoiding flakiness from random VM slowness in CI). hypothesis.settings.register_profile( From 1f481fd3275dbc12a88c16129621de19ea20e4ca Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 29 May 2024 14:44:09 +0200 Subject: [PATCH 260/903] gh-119273: Don't run test_ioctl in a process group (#119275) Python test runner no longer runs tests using TTY (ex: test_ioctl) in a process group (using setsid()). Previously, tests using TTY were skipped. --- Lib/test/libregrtest/run_workers.py | 10 ++++++++-- Lib/test/libregrtest/worker.py | 8 +++++++- .../2024-05-20-18-06-31.gh-issue-119273.hf-yhX.rst | 3 +++ 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-05-20-18-06-31.gh-issue-119273.hf-yhX.rst diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 235047cf2e563c..a71050e66db3bd 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -142,14 +142,20 @@ def _kill(self) -> None: return self._killed = True - if USE_PROCESS_GROUP: + use_killpg = USE_PROCESS_GROUP + if use_killpg: + parent_sid = os.getsid(0) + sid = os.getsid(popen.pid) + use_killpg = (sid != parent_sid) + + if use_killpg: what = f"{self} process group" else: what = f"{self} process" print(f"Kill {what}", file=sys.stderr, flush=True) try: - if USE_PROCESS_GROUP: + if use_killpg: os.killpg(popen.pid, signal.SIGKILL) else: popen.kill() diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index f8b8e45eca3276..15d32b5baa04d0 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -14,6 +14,9 @@ USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) +NEED_TTY = set(''' + test_ioctl +'''.split()) def create_worker_process(runtests: WorkerRunTests, output_fd: int, @@ -47,7 +50,10 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, close_fds=True, cwd=work_dir, ) - if USE_PROCESS_GROUP: + + # Don't use setsid() in tests using TTY + test_name = runtests.tests[0] + if USE_PROCESS_GROUP and test_name not in NEED_TTY: kwargs['start_new_session'] = True # Pass json_file to the worker process diff --git a/Misc/NEWS.d/next/Tests/2024-05-20-18-06-31.gh-issue-119273.hf-yhX.rst b/Misc/NEWS.d/next/Tests/2024-05-20-18-06-31.gh-issue-119273.hf-yhX.rst new file mode 100644 index 00000000000000..905b4e3a1c9043 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-05-20-18-06-31.gh-issue-119273.hf-yhX.rst @@ -0,0 +1,3 @@ +Python test runner no longer runs tests using TTY (ex: test_ioctl) in a +process group (using ``setsid()``). Previously, tests using TTY were +skipped. Patch by Victor Stinner. From 34f9b3e7244615d2372614b20e10250e68cc8e61 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 29 May 2024 18:43:03 +0300 Subject: [PATCH 261/903] gh-119655: Fix reference leak in the ``_datetimemodule.c`` (gh-119713) --- Modules/_datetimemodule.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index b72a5d3c70b92a..466382b5148509 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -6972,10 +6972,13 @@ _datetime_exec(PyObject *module) } while (0) PyTypeObject *PyDateTime_IsoCalendarDateType = NULL; - CREATE_TYPE(PyDateTime_IsoCalendarDateType, &isocal_spec, &PyTuple_Type); + datetime_state *st = get_datetime_state(); + + if (!st->initialized) { + CREATE_TYPE(PyDateTime_IsoCalendarDateType, &isocal_spec, &PyTuple_Type); + } #undef CREATE_TYPE - datetime_state *st = get_datetime_state(); if (init_state(st, PyDateTime_IsoCalendarDateType) < 0) { goto error; } From 78d697b7d5ec2a6fa046b0e1c34e804f49e750b4 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 29 May 2024 16:51:09 +0100 Subject: [PATCH 262/903] gh-119690: Adds Unicode support for named pipes in _winapi (GH-119717) --- Lib/test/audit-tests.py | 11 ++++++ Lib/test/test_audit.py | 14 +++++++ Lib/test/test_winapi.py | 34 ++++++++++++++++- ...-05-29-11-06-12.gh-issue-119690.8q6e1p.rst | 1 + Modules/_winapi.c | 37 +++++++++---------- Modules/clinic/_winapi.c.h | 24 +++++++----- 6 files changed, 92 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-05-29-11-06-12.gh-issue-119690.8q6e1p.rst diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index de7d0da560a1c7..b9021467817f27 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -556,6 +556,17 @@ def hook(event, args): sys.monitoring.register_callback(1, 1, None) +def test_winapi_createnamedpipe(pipe_name): + import _winapi + + def hook(event, args): + if event == "_winapi.CreateNamedPipe": + print(event, args) + + sys.addaudithook(hook) + _winapi.CreateNamedPipe(pipe_name, _winapi.PIPE_ACCESS_DUPLEX, 8, 2, 0, 0, 0, 0) + + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index e163c7ad25cc7b..321d4f9abce8c7 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -291,6 +291,20 @@ def test_sys_monitoring_register_callback(self): self.assertEqual(actual, expected) + def test_winapi_createnamedpipe(self): + winapi = import_helper.import_module("_winapi") + + pipe_name = r"\\.\pipe\LOCAL\test_winapi_createnamed_pipe" + returncode, events, stderr = self.run_python("test_winapi_createnamedpipe", pipe_name) + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("_winapi.CreateNamedPipe", f"({pipe_name!r}, 3, 8)")] + + self.assertEqual(actual, expected) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py index 2ac6f3621710cd..73b8228ddc97ba 100644 --- a/Lib/test/test_winapi.py +++ b/Lib/test/test_winapi.py @@ -7,7 +7,7 @@ import threading import time import unittest -from test.support import import_helper +from test.support import import_helper, os_helper _winapi = import_helper.import_module('_winapi', required_on=['win']) @@ -127,3 +127,35 @@ def test_getshortpathname(self): # Should contain "PROGRA~" but we can't predict the number self.assertIsNotNone(re.match(r".\:\\PROGRA~\d", actual.upper()), actual) + + def test_namedpipe(self): + pipe_name = rf"\\.\pipe\LOCAL\{os_helper.TESTFN}" + + # Pipe does not exist, so this raises + with self.assertRaises(FileNotFoundError): + _winapi.WaitNamedPipe(pipe_name, 0) + + pipe = _winapi.CreateNamedPipe( + pipe_name, + _winapi.PIPE_ACCESS_DUPLEX, + 8, # 8=PIPE_REJECT_REMOTE_CLIENTS + 2, # two instances available + 32, 32, 0, 0) + self.addCleanup(_winapi.CloseHandle, pipe) + + # Pipe instance is available, so this passes + _winapi.WaitNamedPipe(pipe_name, 0) + + with open(pipe_name, 'w+b') as pipe2: + # No instances available, so this times out + # (WinError 121 does not get mapped to TimeoutError) + with self.assertRaises(OSError): + _winapi.WaitNamedPipe(pipe_name, 0) + + _winapi.WriteFile(pipe, b'testdata') + self.assertEqual(b'testdata', pipe2.read(8)) + + self.assertEqual((b'', 0), _winapi.PeekNamedPipe(pipe, 8)[:2]) + pipe2.write(b'testdata') + pipe2.flush() + self.assertEqual((b'testdata', 8), _winapi.PeekNamedPipe(pipe, 8)[:2]) diff --git a/Misc/NEWS.d/next/Windows/2024-05-29-11-06-12.gh-issue-119690.8q6e1p.rst b/Misc/NEWS.d/next/Windows/2024-05-29-11-06-12.gh-issue-119690.8q6e1p.rst new file mode 100644 index 00000000000000..84dd2161aa1db8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-05-29-11-06-12.gh-issue-119690.8q6e1p.rst @@ -0,0 +1 @@ +Adds Unicode support and fixes audit events for ``_winapi.CreateNamedPipe``. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index cd5dd503abe61f..8794d568e92a36 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -224,7 +224,6 @@ create_converter('LPCVOID', '" F_POINTER "') create_converter('BOOL', 'i') # F_BOOL used previously (always 'i') create_converter('DWORD', 'k') # F_DWORD is always "k" (which is much shorter) -create_converter('LPCTSTR', 's') create_converter('UINT', 'I') # F_UINT used previously (always 'I') class LPCWSTR_converter(Py_UNICODE_converter): @@ -259,7 +258,7 @@ class LPVOID_return_converter(CReturnConverter): data.return_conversion.append( 'return_value = HANDLE_TO_PYNUM(_return_value);\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=ef52a757a1830d92]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=da0a4db751936ee7]*/ #include "clinic/_winapi.c.h" @@ -530,7 +529,7 @@ _winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name, { HANDLE handle; - if (PySys_Audit("_winapi.CreateFile", "uIIII", + if (PySys_Audit("_winapi.CreateFile", "ukkkk", file_name, desired_access, share_mode, creation_disposition, flags_and_attributes) < 0) { return INVALID_HANDLE_VALUE; @@ -777,7 +776,7 @@ _winapi_CreateMutexW_impl(PyObject *module, /*[clinic input] _winapi.CreateNamedPipe -> HANDLE - name: LPCTSTR + name: LPCWSTR open_mode: DWORD pipe_mode: DWORD max_instances: DWORD @@ -789,25 +788,25 @@ _winapi.CreateNamedPipe -> HANDLE [clinic start generated code]*/ static HANDLE -_winapi_CreateNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD open_mode, +_winapi_CreateNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD open_mode, DWORD pipe_mode, DWORD max_instances, DWORD out_buffer_size, DWORD in_buffer_size, DWORD default_timeout, LPSECURITY_ATTRIBUTES security_attributes) -/*[clinic end generated code: output=80f8c07346a94fbc input=5a73530b84d8bc37]*/ +/*[clinic end generated code: output=7d6fde93227680ba input=5bd4e4a55639ee02]*/ { HANDLE handle; - if (PySys_Audit("_winapi.CreateNamedPipe", "uII", + if (PySys_Audit("_winapi.CreateNamedPipe", "ukk", name, open_mode, pipe_mode) < 0) { return INVALID_HANDLE_VALUE; } Py_BEGIN_ALLOW_THREADS - handle = CreateNamedPipe(name, open_mode, pipe_mode, - max_instances, out_buffer_size, - in_buffer_size, default_timeout, - security_attributes); + handle = CreateNamedPipeW(name, open_mode, pipe_mode, + max_instances, out_buffer_size, + in_buffer_size, default_timeout, + security_attributes); Py_END_ALLOW_THREADS if (handle == INVALID_HANDLE_VALUE) @@ -1790,7 +1789,7 @@ _winapi_OpenEventW_impl(PyObject *module, DWORD desired_access, { HANDLE handle; - if (PySys_Audit("_winapi.OpenEventW", "Iu", desired_access, name) < 0) { + if (PySys_Audit("_winapi.OpenEventW", "ku", desired_access, name) < 0) { return INVALID_HANDLE_VALUE; } @@ -1821,7 +1820,7 @@ _winapi_OpenMutexW_impl(PyObject *module, DWORD desired_access, { HANDLE handle; - if (PySys_Audit("_winapi.OpenMutexW", "Iu", desired_access, name) < 0) { + if (PySys_Audit("_winapi.OpenMutexW", "ku", desired_access, name) < 0) { return INVALID_HANDLE_VALUE; } @@ -1882,7 +1881,7 @@ _winapi_OpenProcess_impl(PyObject *module, DWORD desired_access, { HANDLE handle; - if (PySys_Audit("_winapi.OpenProcess", "II", + if (PySys_Audit("_winapi.OpenProcess", "kk", process_id, desired_access) < 0) { return INVALID_HANDLE_VALUE; } @@ -2236,19 +2235,19 @@ _winapi_VirtualQuerySize_impl(PyObject *module, LPCVOID address) /*[clinic input] _winapi.WaitNamedPipe - name: LPCTSTR + name: LPCWSTR timeout: DWORD / [clinic start generated code]*/ static PyObject * -_winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout) -/*[clinic end generated code: output=c2866f4439b1fe38 input=36fc781291b1862c]*/ +_winapi_WaitNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD timeout) +/*[clinic end generated code: output=e161e2e630b3e9c2 input=099a4746544488fa]*/ { BOOL success; Py_BEGIN_ALLOW_THREADS - success = WaitNamedPipe(name, timeout); + success = WaitNamedPipeW(name, timeout); Py_END_ALLOW_THREADS if (!success) @@ -2917,7 +2916,7 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, HRESULT hr; COPYFILE2_EXTENDED_PARAMETERS params = { sizeof(COPYFILE2_EXTENDED_PARAMETERS) }; - if (PySys_Audit("_winapi.CopyFile2", "uuI", + if (PySys_Audit("_winapi.CopyFile2", "uuk", existing_file_name, new_file_name, flags) < 0) { return NULL; } diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 9acb2dc4fe7eba..b0c54fc809f4c1 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -445,7 +445,7 @@ PyDoc_STRVAR(_winapi_CreateNamedPipe__doc__, {"CreateNamedPipe", _PyCFunction_CAST(_winapi_CreateNamedPipe), METH_FASTCALL, _winapi_CreateNamedPipe__doc__}, static HANDLE -_winapi_CreateNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD open_mode, +_winapi_CreateNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD open_mode, DWORD pipe_mode, DWORD max_instances, DWORD out_buffer_size, DWORD in_buffer_size, DWORD default_timeout, @@ -455,7 +455,7 @@ static PyObject * _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR name; + LPCWSTR name = NULL; DWORD open_mode; DWORD pipe_mode; DWORD max_instances; @@ -465,8 +465,8 @@ _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t narg LPSECURITY_ATTRIBUTES security_attributes; HANDLE _return_value; - if (!_PyArg_ParseStack(args, nargs, "skkkkkk" F_POINTER ":CreateNamedPipe", - &name, &open_mode, &pipe_mode, &max_instances, &out_buffer_size, &in_buffer_size, &default_timeout, &security_attributes)) { + if (!_PyArg_ParseStack(args, nargs, "O&kkkkkk" F_POINTER ":CreateNamedPipe", + _PyUnicode_WideCharString_Converter, &name, &open_mode, &pipe_mode, &max_instances, &out_buffer_size, &in_buffer_size, &default_timeout, &security_attributes)) { goto exit; } _return_value = _winapi_CreateNamedPipe_impl(module, name, open_mode, pipe_mode, max_instances, out_buffer_size, in_buffer_size, default_timeout, security_attributes); @@ -479,6 +479,9 @@ _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t narg return_value = HANDLE_TO_PYNUM(_return_value); exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + return return_value; } @@ -1660,22 +1663,25 @@ PyDoc_STRVAR(_winapi_WaitNamedPipe__doc__, {"WaitNamedPipe", _PyCFunction_CAST(_winapi_WaitNamedPipe), METH_FASTCALL, _winapi_WaitNamedPipe__doc__}, static PyObject * -_winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout); +_winapi_WaitNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD timeout); static PyObject * _winapi_WaitNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR name; + LPCWSTR name = NULL; DWORD timeout; - if (!_PyArg_ParseStack(args, nargs, "sk:WaitNamedPipe", - &name, &timeout)) { + if (!_PyArg_ParseStack(args, nargs, "O&k:WaitNamedPipe", + _PyUnicode_WideCharString_Converter, &name, &timeout)) { goto exit; } return_value = _winapi_WaitNamedPipe_impl(module, name, timeout); exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + return return_value; } @@ -2118,4 +2124,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=ed94a2482ede3744 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2304c62187a90140 input=a9049054013a1b77]*/ From 659cb7e6b8e83e1541fc27fd29d4846e940b600e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=81ajszczak?= Date: Wed, 29 May 2024 18:39:34 +0200 Subject: [PATCH 263/903] gh-119721: Integrate documentation fixes into heapq module docstring. (gh-119722) --- Lib/heapq.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/heapq.py b/Lib/heapq.py index c53cb5537db1a4..9649da251f2a83 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -78,7 +78,7 @@ not "better" than the last 0'th element you extracted. This is especially useful in simulation contexts, where the tree holds all incoming events, and the "win" condition means the smallest scheduled -time. When an event schedule other events for execution, they are +time. When an event schedules other events for execution, they are scheduled into the future, so they can easily go into the heap. So, a heap is a good structure for implementing schedulers (this is what I used for my MIDI sequencer :-). @@ -91,14 +91,14 @@ Heaps are also very useful in big disk sorts. You most probably all know that a big sort implies producing "runs" (which are pre-sorted -sequences, which size is usually related to the amount of CPU memory), +sequences, whose size is usually related to the amount of CPU memory), followed by a merging passes for these runs, which merging is often very cleverly organised[1]. It is very important that the initial sort produces the longest runs possible. Tournaments are a good way -to that. If, using all the memory available to hold a tournament, you -replace and percolate items that happen to fit the current run, you'll -produce runs which are twice the size of the memory for random input, -and much better for input fuzzily ordered. +to achieve that. If, using all the memory available to hold a +tournament, you replace and percolate items that happen to fit the +current run, you'll produce runs which are twice the size of the +memory for random input, and much better for input fuzzily ordered. Moreover, if you output the 0'th item on disk and get an input which may not fit in the current tournament (because the value "wins" over From c8b45a385ab85c3f3463b1e7394c515e892dfba7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 29 May 2024 12:43:19 -0400 Subject: [PATCH 264/903] gh-118673: Remove shebang and executable bits from stdlib modules. (#119658) * gh-118673: Remove shebang and executable bits from stdlib modules. * Removed shebangs and exe bits on turtledemo scripts. The setting was inappropriate for '__main__' and inconsistent across the other modules. The scripts can still be executed directly by invoking with the desired interpreter. --- Lib/base64.py | 2 -- Lib/cProfile.py | 2 -- Lib/pdb.py | 2 -- Lib/platform.py | 2 -- Lib/profile.py | 1 - Lib/pydoc.py | 1 - Lib/quopri.py | 2 -- Lib/smtplib.py | 2 -- Lib/tabnanny.py | 2 -- Lib/tarfile.py | 1 - Lib/timeit.py | 2 -- Lib/trace.py | 2 -- Lib/turtledemo/__main__.py | 2 -- Lib/turtledemo/bytedesign.py | 1 - Lib/turtledemo/clock.py | 1 - Lib/turtledemo/forest.py | 1 - Lib/turtledemo/fractalcurves.py | 1 - Lib/turtledemo/lindenmayer.py | 1 - Lib/turtledemo/minimal_hanoi.py | 1 - Lib/turtledemo/paint.py | 1 - Lib/turtledemo/peace.py | 1 - Lib/turtledemo/penrose.py | 1 - Lib/turtledemo/planet_and_moon.py | 1 - Lib/turtledemo/sorting_animate.py | 1 - Lib/turtledemo/tree.py | 1 - Lib/turtledemo/yinyang.py | 1 - Lib/webbrowser.py | 1 - .../next/Library/2024-05-06-17-39-52.gh-issue-118673.sTXBit.rst | 1 + 28 files changed, 1 insertion(+), 37 deletions(-) mode change 100755 => 100644 Lib/base64.py mode change 100755 => 100644 Lib/cProfile.py mode change 100755 => 100644 Lib/pdb.py mode change 100755 => 100644 Lib/platform.py mode change 100755 => 100644 Lib/profile.py mode change 100755 => 100644 Lib/pydoc.py mode change 100755 => 100644 Lib/quopri.py mode change 100755 => 100644 Lib/smtplib.py mode change 100755 => 100644 Lib/tabnanny.py mode change 100755 => 100644 Lib/tarfile.py mode change 100755 => 100644 Lib/timeit.py mode change 100755 => 100644 Lib/trace.py mode change 100755 => 100644 Lib/turtledemo/__main__.py mode change 100755 => 100644 Lib/turtledemo/bytedesign.py mode change 100755 => 100644 Lib/turtledemo/clock.py mode change 100755 => 100644 Lib/turtledemo/forest.py mode change 100755 => 100644 Lib/turtledemo/fractalcurves.py mode change 100755 => 100644 Lib/turtledemo/lindenmayer.py mode change 100755 => 100644 Lib/turtledemo/minimal_hanoi.py mode change 100755 => 100644 Lib/turtledemo/paint.py mode change 100755 => 100644 Lib/turtledemo/peace.py mode change 100755 => 100644 Lib/turtledemo/penrose.py mode change 100755 => 100644 Lib/turtledemo/planet_and_moon.py mode change 100755 => 100644 Lib/turtledemo/sorting_animate.py mode change 100755 => 100644 Lib/turtledemo/tree.py mode change 100755 => 100644 Lib/turtledemo/yinyang.py mode change 100755 => 100644 Lib/webbrowser.py create mode 100644 Misc/NEWS.d/next/Library/2024-05-06-17-39-52.gh-issue-118673.sTXBit.rst diff --git a/Lib/base64.py b/Lib/base64.py old mode 100755 new mode 100644 index 5a7e790a193380..61be4fb856e92c --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module diff --git a/Lib/cProfile.py b/Lib/cProfile.py old mode 100755 new mode 100644 index 9c132372dc4ee0..e7c868b8d55543 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Python interface for the 'lsprof' profiler. Compatible with the 'profile' module. """ diff --git a/Lib/pdb.py b/Lib/pdb.py old mode 100755 new mode 100644 index 0f791d71950099..ba84a29aa2f669 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """ The Python Debugger Pdb ======================= diff --git a/Lib/platform.py b/Lib/platform.py old mode 100755 new mode 100644 index 5958382276e79c..a4fd2463f15a6c --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """ This module tries to retrieve as much platform-identifying data as possible. It makes this information available via function APIs. diff --git a/Lib/profile.py b/Lib/profile.py old mode 100755 new mode 100644 index f2f8c2f21333e0..a5afb12c9d121a --- a/Lib/profile.py +++ b/Lib/profile.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python3 # # Class for profiling python code. rev 1.0 6/2/94 # diff --git a/Lib/pydoc.py b/Lib/pydoc.py old mode 100755 new mode 100644 index 55ccf2152c26cb..5d854c50f40d6e --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Generate Python documentation in HTML or text for interactive use. At the Python interactive prompt, calling help(thing) on a Python object diff --git a/Lib/quopri.py b/Lib/quopri.py old mode 100755 new mode 100644 index f36cf7b3951cda..129fd2f5c7c28a --- a/Lib/quopri.py +++ b/Lib/quopri.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Conversions to/from quoted-printable transport encoding as per RFC 1521.""" # (Dec 1991 version). diff --git a/Lib/smtplib.py b/Lib/smtplib.py old mode 100755 new mode 100644 index 75163f75781d19..84d6d858e7dec1 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - '''SMTP/ESMTP client class. This should follow RFC 821 (SMTP), RFC 1869 (ESMTP), RFC 2554 (SMTP diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py old mode 100755 new mode 100644 index e2ac6837f157d5..7e56d4a48d1d00 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """The Tab Nanny despises ambiguous indentation. She knows no mercy. tabnanny -- Detection of ambiguous indentation diff --git a/Lib/tarfile.py b/Lib/tarfile.py old mode 100755 new mode 100644 index f817b57ab1640e..d5d8a469779f50 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 #------------------------------------------------------------------- # tarfile.py #------------------------------------------------------------------- diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100755 new mode 100644 index 02cfafaf36e5d1..c106e0f67356da --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Tool for measuring execution time of small code snippets. This module avoids a number of common traps for measuring execution diff --git a/Lib/trace.py b/Lib/trace.py old mode 100755 new mode 100644 index 8550475e3a78c8..bb3d34fd8d6550 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # portions copyright 2001, Autonomous Zones Industries, Inc., all rights... # err... reserved and offered to the public under the terms of the # Python 2.2 license. diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py old mode 100755 new mode 100644 index 731f98b02b17a7..06a64081a896b5 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """ ---------------------------------------------- turtleDemo - Help diff --git a/Lib/turtledemo/bytedesign.py b/Lib/turtledemo/bytedesign.py old mode 100755 new mode 100644 index 1b7452b512c6eb..476cdaabfceab1 --- a/Lib/turtledemo/bytedesign.py +++ b/Lib/turtledemo/bytedesign.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_bytedesign.py diff --git a/Lib/turtledemo/clock.py b/Lib/turtledemo/clock.py old mode 100755 new mode 100644 index 9f8585bd11e053..fd3b3992d466bf --- a/Lib/turtledemo/clock.py +++ b/Lib/turtledemo/clock.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: cp1252 -*- """ turtle-example-suite: diff --git a/Lib/turtledemo/forest.py b/Lib/turtledemo/forest.py old mode 100755 new mode 100644 index 55b7da947d2476..cac553223828db --- a/Lib/turtledemo/forest.py +++ b/Lib/turtledemo/forest.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtlegraphics-example-suite: tdemo_forest.py diff --git a/Lib/turtledemo/fractalcurves.py b/Lib/turtledemo/fractalcurves.py old mode 100755 new mode 100644 index 54ade96a0ad05e..fda193e06fedee --- a/Lib/turtledemo/fractalcurves.py +++ b/Lib/turtledemo/fractalcurves.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_fractalCurves.py diff --git a/Lib/turtledemo/lindenmayer.py b/Lib/turtledemo/lindenmayer.py old mode 100755 new mode 100644 index 3925f25da61870..7c7a84796c3c28 --- a/Lib/turtledemo/lindenmayer.py +++ b/Lib/turtledemo/lindenmayer.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: xtx_lindenmayer_indian.py diff --git a/Lib/turtledemo/minimal_hanoi.py b/Lib/turtledemo/minimal_hanoi.py old mode 100755 new mode 100644 index 4a432f2b2908d5..08d8b630fec3b4 --- a/Lib/turtledemo/minimal_hanoi.py +++ b/Lib/turtledemo/minimal_hanoi.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_minimal_hanoi.py diff --git a/Lib/turtledemo/paint.py b/Lib/turtledemo/paint.py old mode 100755 new mode 100644 index fc6852a20082f5..6e63d004454589 --- a/Lib/turtledemo/paint.py +++ b/Lib/turtledemo/paint.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_paint.py diff --git a/Lib/turtledemo/peace.py b/Lib/turtledemo/peace.py old mode 100755 new mode 100644 index e2ba9288d9e42e..fd6abe390ef198 --- a/Lib/turtledemo/peace.py +++ b/Lib/turtledemo/peace.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_peace.py diff --git a/Lib/turtledemo/penrose.py b/Lib/turtledemo/penrose.py old mode 100755 new mode 100644 index 045722a2286061..ac12c899d3844e --- a/Lib/turtledemo/penrose.py +++ b/Lib/turtledemo/penrose.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ xturtle-example-suite: xtx_kites_and_darts.py diff --git a/Lib/turtledemo/planet_and_moon.py b/Lib/turtledemo/planet_and_moon.py old mode 100755 new mode 100644 index 021ff99383aa65..c0e2c5b79e173e --- a/Lib/turtledemo/planet_and_moon.py +++ b/Lib/turtledemo/planet_and_moon.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_planets_and_moon.py diff --git a/Lib/turtledemo/sorting_animate.py b/Lib/turtledemo/sorting_animate.py old mode 100755 new mode 100644 index d25a0ab6cebdc0..55735cd7001278 --- a/Lib/turtledemo/sorting_animate.py +++ b/Lib/turtledemo/sorting_animate.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ sorting_animation.py diff --git a/Lib/turtledemo/tree.py b/Lib/turtledemo/tree.py old mode 100755 new mode 100644 index 98a20da7f15c11..12729e23688a48 --- a/Lib/turtledemo/tree.py +++ b/Lib/turtledemo/tree.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_tree.py diff --git a/Lib/turtledemo/yinyang.py b/Lib/turtledemo/yinyang.py old mode 100755 new mode 100644 index 11d1f47cae2549..791060d17e6b6a --- a/Lib/turtledemo/yinyang.py +++ b/Lib/turtledemo/yinyang.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ turtle-example-suite: tdemo_yinyang.py diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py old mode 100755 new mode 100644 index b7fbc41853ea65..6fca257c02664f --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python3 """Interfaces for launching and remotely controlling web browsers.""" # Maintained by Georg Brandl. diff --git a/Misc/NEWS.d/next/Library/2024-05-06-17-39-52.gh-issue-118673.sTXBit.rst b/Misc/NEWS.d/next/Library/2024-05-06-17-39-52.gh-issue-118673.sTXBit.rst new file mode 100644 index 00000000000000..f0a87d2a91df3c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-06-17-39-52.gh-issue-118673.sTXBit.rst @@ -0,0 +1 @@ +Removed executable bits and shebang from stdlib modules. From 0751511d24295c39fdf2f5b2255e3fa3d796ce4d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 29 May 2024 20:08:27 +0300 Subject: [PATCH 265/903] gh-93963: Remove deprecated names from importlib.abc (#119720) Co-authored-by: Jason R. Coombs --- Doc/whatsnew/3.14.rst | 34 +++++++++++++------ Lib/importlib/abc.py | 15 -------- ...4-05-29-12-42-40.gh-issue-93963.cb1oJS.rst | 2 ++ 3 files changed, 26 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-29-12-42-40.gh-issue-93963.cb1oJS.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index bc7fe64e68bb18..8c37825430c2cf 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -90,7 +90,7 @@ ast --- Added :func:`ast.compare` for comparing two ASTs. -(Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`) +(Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.) @@ -108,6 +108,13 @@ Deprecated Removed ======= +argparse +-------- + +* Remove the *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction`. + They were deprecated since 3.12. + ast --- @@ -137,27 +144,34 @@ ast (Contributed by Alex Waygood in :gh:`119562`.) - -argparse --------- - -* Remove the *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction`. - They were deprecated since 3.12. - collections.abc --------------- * Remove :class:`!collections.abc.ByteString`. It had previously raised a :exc:`DeprecationWarning` since Python 3.12. - email ----- * Remove the *isdst* parameter from :func:`email.utils.localtime`. (Contributed by Hugo van Kemenade in :gh:`118798`.) +importlib +--------- + +* Remove deprecated :mod:`importlib.abc` classes: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + itertools --------- diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 37fef357fe2c0c..b6b2c791a3b03f 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -15,8 +15,6 @@ import abc import warnings -from .resources import abc as _resources_abc - __all__ = [ 'Loader', 'MetaPathFinder', 'PathEntryFinder', @@ -25,19 +23,6 @@ ] -def __getattr__(name): - """ - For backwards compatibility, continue to make names - from _resources_abc available through this module. #93963 - """ - if name in _resources_abc.__all__: - obj = getattr(_resources_abc, name) - warnings._deprecated(f"{__name__}.{name}", remove=(3, 14)) - globals()[name] = obj - return obj - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - - def _register(abstract_cls, *classes): for cls in classes: abstract_cls.register(cls) diff --git a/Misc/NEWS.d/next/Library/2024-05-29-12-42-40.gh-issue-93963.cb1oJS.rst b/Misc/NEWS.d/next/Library/2024-05-29-12-42-40.gh-issue-93963.cb1oJS.rst new file mode 100644 index 00000000000000..d093c8e35a5994 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-29-12-42-40.gh-issue-93963.cb1oJS.rst @@ -0,0 +1,2 @@ +Remove deprecated names from ``importlib.abc`` as found in +``importlib.resources.abc``. From bf4ff3ad2e362801e87c85fffd9e140b774cef26 Mon Sep 17 00:00:00 2001 From: Aditya Borikar Date: Wed, 29 May 2024 11:26:22 -0600 Subject: [PATCH 266/903] gh-119260: Clarify is_dataclass Behavior for Subclasses in Documentation and Tests (#119480) Co-authored-by: Carl Meyer --- Doc/library/dataclasses.rst | 4 ++-- Lib/test/test_dataclasses/__init__.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index cf707ca5b6802d..fcb5e8bad295a0 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -461,8 +461,8 @@ Module contents .. function:: is_dataclass(obj) - Return ``True`` if its parameter is a dataclass or an instance of one, - otherwise return ``False``. + Return ``True`` if its parameter is a dataclass (including subclasses of a + dataclass) or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 04dd9f3265bb33..ffb8bbe75c504f 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -1547,6 +1547,24 @@ class A(types.GenericAlias): self.assertTrue(is_dataclass(type(a))) self.assertTrue(is_dataclass(a)) + def test_is_dataclass_inheritance(self): + @dataclass + class X: + y: int + + class Z(X): + pass + + self.assertTrue(is_dataclass(X), "X should be a dataclass") + self.assertTrue( + is_dataclass(Z), + "Z should be a dataclass because it inherits from X", + ) + z_instance = Z(y=5) + self.assertTrue( + is_dataclass(z_instance), + "z_instance should be a dataclass because it is an instance of Z", + ) def test_helper_fields_with_class_instance(self): # Check that we can call fields() on either a class or instance, From fcca08ec2f48f4ba5ba1d4690fb39b1efe630944 Mon Sep 17 00:00:00 2001 From: Wim Jeantine-Glenn Date: Wed, 29 May 2024 12:46:20 -0500 Subject: [PATCH 267/903] gh-119594: Improve pow(fraction.Fraction(), b, modulo) error message (#119593) If one calls pow(fractions.Fraction, x, module) with modulo not None, the error message now says that the types are incompatible rather than saying pow only takes 2 arguments. Implemented by having fractions.Fraction __pow__ accept optional modulo argument and return NotImplemented if not None. pow() then raises with appropriate message. --------- Co-authored-by: Mark Dickinson --- Lib/fractions.py | 4 +++- Lib/test/test_fractions.py | 6 ++++++ .../Library/2024-05-26-22-22-51.gh-issue-119594.fnQNM8.rst | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-26-22-22-51.gh-issue-119594.fnQNM8.rst diff --git a/Lib/fractions.py b/Lib/fractions.py index f8c6c9c438c737..f91b4f35eff370 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -848,7 +848,7 @@ def _mod(a, b): __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) - def __pow__(a, b): + def __pow__(a, b, modulo=None): """a ** b If b is not an integer, the result will be a float or complex @@ -856,6 +856,8 @@ def __pow__(a, b): result will be rational. """ + if modulo is not None: + return NotImplemented if isinstance(b, numbers.Rational): if b.denominator == 1: power = b.numerator diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 3a714c64278847..3648a8982a37e0 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1633,6 +1633,12 @@ def test_complex_handling(self): message % ("divmod()", "complex", "Fraction"), divmod, b, a) + def test_three_argument_pow(self): + message = "unsupported operand type(s) for ** or pow(): '%s', '%s', '%s'" + self.assertRaisesMessage(TypeError, + message % ("Fraction", "int", "int"), + pow, F(3), 4, 5) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2024-05-26-22-22-51.gh-issue-119594.fnQNM8.rst b/Misc/NEWS.d/next/Library/2024-05-26-22-22-51.gh-issue-119594.fnQNM8.rst new file mode 100644 index 00000000000000..d2de5273edf571 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-26-22-22-51.gh-issue-119594.fnQNM8.rst @@ -0,0 +1 @@ +If one calls pow(fractions.Fraction, x, module) with modulo not None, the error message now says that the types are incompatible rather than saying pow only takes 2 arguments. Patch by Wim Jeantine-Glenn and Mark Dickinson. From df93f5d4bf9d70036d485666d4dd4f009d37f8b9 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 29 May 2024 18:51:13 +0100 Subject: [PATCH 268/903] gh-119070: Fix py.exe handling of /usr/bin/env commands missing extension (GH-119426) --- Lib/test/test_launcher.py | 8 ++++++++ .../2024-05-22-19-43-29.gh-issue-119070._enton.rst | 3 +++ PC/launcher2.c | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-05-22-19-43-29.gh-issue-119070._enton.rst diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index 2528a51240fbf7..6d358ac6f16a27 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -764,3 +764,11 @@ def test_shebang_command_in_venv(self): with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script: data = self.run_py([script], env=env) self.assertEqual(data["stdout"].strip(), f"{quote(exe)} arg1 {quote(script)}") + + def test_shebang_executable_extension(self): + with self.script('#! /usr/bin/env python3.12') as script: + data = self.run_py([script]) + expect = "# Search PATH for python3.12.exe" + actual = [line.strip() for line in data["stderr"].splitlines() + if line.startswith("# Search PATH")] + self.assertEqual([expect], actual) diff --git a/Misc/NEWS.d/next/Windows/2024-05-22-19-43-29.gh-issue-119070._enton.rst b/Misc/NEWS.d/next/Windows/2024-05-22-19-43-29.gh-issue-119070._enton.rst new file mode 100644 index 00000000000000..aab26f57209864 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-05-22-19-43-29.gh-issue-119070._enton.rst @@ -0,0 +1,3 @@ +Fixes ``py.exe`` handling of shebangs like ``/usr/bin/env python3.12``, +which were previously interpreted as ``python3.exe`` instead of +``python3.12.exe``. diff --git a/PC/launcher2.c b/PC/launcher2.c index 98231613efb26f..b372044e353202 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -853,7 +853,7 @@ searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) } wchar_t filename[MAXLEN]; - if (wcsncpy_s(filename, MAXLEN, command, lastDot)) { + if (wcsncpy_s(filename, MAXLEN, command, commandLength)) { return RC_BAD_VIRTUAL_PATH; } From c22323cd1c200ca1b22c47af95f67c4b2d661fe7 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 29 May 2024 15:26:04 -0400 Subject: [PATCH 269/903] gh-119525: Fix deadlock with `_PyType_Lookup` and the GIL (#119527) The deadlock only affected the free-threaded build and only occurred when the GIL was enabled at runtime. The `Py_DECREF(old_name)` call might temporarily release the GIL while holding the type seqlock. Another thread may spin trying to acquire the seqlock while holding the GIL. The deadlock occurred roughly 1 in ~1,000 runs of `pool_in_threads.py` from `test_multiprocessing_pool_circular_import`. --- .../2024-05-24-21-04-00.gh-issue-119525.zLFLf1.rst | 2 ++ Objects/typeobject.c | 11 +++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-04-00.gh-issue-119525.zLFLf1.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-04-00.gh-issue-119525.zLFLf1.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-04-00.gh-issue-119525.zLFLf1.rst new file mode 100644 index 00000000000000..83c29a16e572d7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-04-00.gh-issue-119525.zLFLf1.rst @@ -0,0 +1,2 @@ +Fix deadlock involving ``_PyType_Lookup()`` cache in the free-threaded build +when the GIL is dynamically enabled at runtime. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9d849d83082ccd..290306cdb677e5 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5169,7 +5169,7 @@ is_dunder_name(PyObject *name) return 0; } -static void +static PyObject * update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) { _Py_atomic_store_uint32_relaxed(&entry->version, version_tag); @@ -5180,7 +5180,7 @@ update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int versio // exact unicode object or Py_None so it's safe to do so. PyObject *old_name = entry->name; _Py_atomic_store_ptr_relaxed(&entry->name, Py_NewRef(name)); - Py_DECREF(old_name); + return old_name; } #if Py_GIL_DISABLED @@ -5200,10 +5200,12 @@ update_cache_gil_disabled(struct type_cache_entry *entry, PyObject *name, return; } - update_cache(entry, name, version_tag, value); + PyObject *old_value = update_cache(entry, name, version_tag, value); // Then update sequence to the next valid value _PySeqLock_UnlockWrite(&entry->sequence); + + Py_DECREF(old_value); } #endif @@ -5315,7 +5317,8 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name) #if Py_GIL_DISABLED update_cache_gil_disabled(entry, name, version, res); #else - update_cache(entry, name, version, res); + PyObject *old_value = update_cache(entry, name, version, res); + Py_DECREF(old_value); #endif } return res; From a150679f90c6e3f017bd75cac3b8f727063cc4aa Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 29 May 2024 21:11:30 +0100 Subject: [PATCH 270/903] GH-89727: Partially fix `shutil.rmtree()` recursion error on deep trees (#119634) Make `shutil._rmtree_unsafe()` call `os.walk()`, which is implemented without recursion. `shutil._rmtree_safe_fd()` is not affected and can still raise a recursion error. Co-authored-by: Jelle Zijlstra --- Lib/os.py | 9 ++++- Lib/shutil.py | 38 ++++++------------- Lib/test/test_shutil.py | 11 ++++++ ...4-05-29-20-42-17.gh-issue-89727.5lPTTW.rst | 3 ++ 4 files changed, 33 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-29-20-42-17.gh-issue-89727.5lPTTW.rst diff --git a/Lib/os.py b/Lib/os.py index 7661ce68ca3be2..ae9e646361e82c 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -281,6 +281,10 @@ def renames(old, new): __all__.extend(["makedirs", "removedirs", "renames"]) +# Private sentinel that makes walk() classify all symlinks and junctions as +# regular files. +_walk_symlinks_as_files = object() + def walk(top, topdown=True, onerror=None, followlinks=False): """Directory tree generator. @@ -382,7 +386,10 @@ def walk(top, topdown=True, onerror=None, followlinks=False): break try: - is_dir = entry.is_dir() + if followlinks is _walk_symlinks_as_files: + is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() + else: + is_dir = entry.is_dir() except OSError: # If is_dir() raises an OSError, consider the entry not to # be a directory, same behaviour as os.path.isdir(). diff --git a/Lib/shutil.py b/Lib/shutil.py index c9b4da34b1e19b..03a9d756030430 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -606,37 +606,21 @@ def _rmtree_islink(st): # version vulnerable to race conditions def _rmtree_unsafe(path, onexc): - try: - with os.scandir(path) as scandir_it: - entries = list(scandir_it) - except FileNotFoundError: - return - except OSError as err: - onexc(os.scandir, path, err) - entries = [] - for entry in entries: - fullname = entry.path - try: - is_dir = entry.is_dir(follow_symlinks=False) - except FileNotFoundError: - continue - except OSError: - is_dir = False - - if is_dir and not entry.is_junction(): + def onerror(err): + if not isinstance(err, FileNotFoundError): + onexc(os.scandir, err.filename, err) + results = os.walk(path, topdown=False, onerror=onerror, followlinks=os._walk_symlinks_as_files) + for dirpath, dirnames, filenames in results: + for name in dirnames: + fullname = os.path.join(dirpath, name) try: - if entry.is_symlink(): - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or entry.is_dir above. - raise OSError("Cannot call rmtree on a symbolic link") + os.rmdir(fullname) except FileNotFoundError: continue except OSError as err: - onexc(os.path.islink, fullname, err) - continue - _rmtree_unsafe(fullname, onexc) - else: + onexc(os.rmdir, fullname, err) + for name in filenames: + fullname = os.path.join(dirpath, name) try: os.unlink(fullname) except FileNotFoundError: diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index df9e7a660bf29e..01f139073dcd97 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -741,6 +741,17 @@ def _onexc(fn, path, exc): shutil.rmtree(TESTFN) raise + @unittest.skipIf(shutil._use_fd_functions, "fd-based functions remain unfixed (GH-89727)") + def test_rmtree_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = os.path.join(TESTFN, *(['d'] * directory_depth)) + os.makedirs(base) + + with support.infinite_recursion(recursion_limit): + shutil.rmtree(TESTFN) + class TestCopyTree(BaseTest, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-05-29-20-42-17.gh-issue-89727.5lPTTW.rst b/Misc/NEWS.d/next/Library/2024-05-29-20-42-17.gh-issue-89727.5lPTTW.rst new file mode 100644 index 00000000000000..3b73d2789fd6f9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-29-20-42-17.gh-issue-89727.5lPTTW.rst @@ -0,0 +1,3 @@ +Partially fix issue with :func:`shutil.rmtree` where a :exc:`RecursionError` +is raised on deep directory trees. A recursion error is no longer raised +when :data:`!rmtree.avoids_symlink_attacks` is false. From 7ff61f51b6f75315291419269295a8ac3933397b Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 29 May 2024 21:51:04 +0100 Subject: [PATCH 271/903] GH-119169: Implement `pathlib.Path.walk()` using `os.walk()` (#119573) For silly reasons, pathlib's generic implementation of `walk()` currently resides in `glob._Globber`. This commit moves it into `pathlib._abc.PathBase.walk()` where it really belongs, and makes `pathlib.Path.walk()` call `os.walk()`. --- Lib/glob.py | 37 ------------------------------------- Lib/pathlib/_abc.py | 32 +++++++++++++++++++++++++++++++- Lib/pathlib/_local.py | 4 +++- 3 files changed, 34 insertions(+), 39 deletions(-) diff --git a/Lib/glob.py b/Lib/glob.py index 920f79ad7e1fe5..fbb1d35aab71fa 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -519,43 +519,6 @@ def select_exists(self, path, exists=False): elif self.lexists(path): yield path - @classmethod - def walk(cls, root, top_down, on_error, follow_symlinks): - """Walk the directory tree from the given root, similar to os.walk(). - """ - paths = [root] - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - try: - with cls.scandir(path) as scandir_it: - dirnames = [] - filenames = [] - if not top_down: - paths.append((path, dirnames, filenames)) - for entry in scandir_it: - name = entry.name - try: - if entry.is_dir(follow_symlinks=follow_symlinks): - if not top_down: - paths.append(cls.parse_entry(entry)) - dirnames.append(name) - else: - filenames.append(name) - except OSError: - filenames.append(name) - except OSError as error: - if on_error is not None: - on_error(error) - else: - if top_down: - yield path, dirnames, filenames - if dirnames: - prefix = cls.add_slash(path) - paths += [cls.concat_path(prefix, d) for d in reversed(dirnames)] - class _StringGlobber(_Globber): lexists = staticmethod(os.path.lexists) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 6b5d9fc2a0c560..d7471b6927331d 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -621,7 +621,37 @@ def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): def walk(self, top_down=True, on_error=None, follow_symlinks=False): """Walk the directory tree from this directory, similar to os.walk().""" - return self._globber.walk(self, top_down, on_error, follow_symlinks) + paths = [self] + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + dirnames = [] + filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) + try: + for child in path.iterdir(): + try: + if child.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(child) + dirnames.append(child.name) + else: + filenames.append(child.name) + except OSError: + filenames.append(child.name) + except OSError as error: + if on_error is not None: + on_error(error) + if not top_down: + while not isinstance(paths.pop(), tuple): + pass + continue + if top_down: + yield path, dirnames, filenames + paths += [path.joinpath(d) for d in reversed(dirnames)] def absolute(self): """Return an absolute version of this path diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 49d9f813c54c23..473fd525768b50 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -672,7 +672,9 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): """Walk the directory tree from this directory, similar to os.walk().""" sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) root_dir = str(self) - results = self._globber.walk(root_dir, top_down, on_error, follow_symlinks) + if not follow_symlinks: + follow_symlinks = os._walk_symlinks_as_files + results = os.walk(root_dir, top_down, on_error, follow_symlinks) for path_str, dirnames, filenames in results: if root_dir == '.': path_str = path_str[2:] From 2cc3502f98bb9aea386ab55443fc077ddcdde91d Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 29 May 2024 17:19:54 -0500 Subject: [PATCH 272/903] subprocess docs: Fix semantically important typo (GH-119752) GH-25416 accidentally replaced a reference to the *stderr* argument of `subprocess.run` with a reference to the *stdin* argument. *stdin* is not affected by the `check_output` option. --- Doc/library/subprocess.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index d77dc8b04d01e4..3a2178e2b7b839 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -52,7 +52,7 @@ underlying :class:`Popen` interface can be used directly. If *capture_output* is true, stdout and stderr will be captured. When used, the internal :class:`Popen` object is automatically created with - *stdout* and *stdin* both set to :data:`~subprocess.PIPE`. + *stdout* and *stderr* both set to :data:`~subprocess.PIPE`. The *stdout* and *stderr* arguments may not be supplied at the same time as *capture_output*. If you wish to capture and combine both streams into one, set *stdout* to :data:`~subprocess.PIPE` From 3c890b503c740767d0eb9a0e74b47f17a1e69452 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 30 May 2024 04:05:36 +0100 Subject: [PATCH 273/903] GH-89727: Fix `os.fwalk()` recursion error on deep trees (#119638) Implement `os.fwalk()` using a list as a stack to avoid emitting recursion errors on deeply nested trees. --- Lib/os.py | 91 +++++++++++-------- Lib/test/test_os.py | 2 - ...4-05-28-00-56-59.gh-issue-89727._bxoL3.rst | 3 + 3 files changed, 56 insertions(+), 40 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst diff --git a/Lib/os.py b/Lib/os.py index ae9e646361e82c..cef5d90a8c23df 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -478,24 +478,52 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) top = fspath(top) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - if not follow_symlinks: - orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - topfd = open(top, O_RDONLY | O_NONBLOCK, dir_fd=dir_fd) - try: - if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and - path.samestat(orig_st, stat(topfd)))): - yield from _fwalk(topfd, top, isinstance(top, bytes), - topdown, onerror, follow_symlinks) - finally: - close(topfd) - - def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): + stack = [(_fwalk_walk, (True, dir_fd, top, top, None))] + isbytes = isinstance(top, bytes) + while stack: + yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) + + # Each item in the _fwalk() stack is a pair (action, args). + _fwalk_walk = 0 # args: (isroot, dirfd, toppath, topname, entry) + _fwalk_yield = 1 # args: (toppath, dirnames, filenames, topfd) + _fwalk_close = 2 # args: dirfd + + def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. + action, value = stack.pop() + if action == _fwalk_close: + close(value) + return + elif action == _fwalk_yield: + yield value + return + assert action == _fwalk_walk + isroot, dirfd, toppath, topname, entry = value + try: + if not follow_symlinks: + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + if entry is None: + orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd) + else: + orig_st = entry.stat(follow_symlinks=False) + topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) + except OSError as err: + if isroot: + raise + if onerror is not None: + onerror(err) + return + stack.append((_fwalk_close, topfd)) + if not follow_symlinks: + if isroot and not st.S_ISDIR(orig_st.st_mode): + return + if not path.samestat(orig_st, stat(topfd)): + return + scandir_it = scandir(topfd) dirs = [] nondirs = [] @@ -521,31 +549,18 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd + else: + stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) - for name in dirs if entries is None else zip(dirs, entries): - try: - if not follow_symlinks: - if topdown: - orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) - else: - assert entries is not None - name, entry = name - orig_st = entry.stat(follow_symlinks=False) - dirfd = open(name, O_RDONLY | O_NONBLOCK, dir_fd=topfd) - except OSError as err: - if onerror is not None: - onerror(err) - continue - try: - if follow_symlinks or path.samestat(orig_st, stat(dirfd)): - dirpath = path.join(toppath, name) - yield from _fwalk(dirfd, dirpath, isbytes, - topdown, onerror, follow_symlinks) - finally: - close(dirfd) - - if not topdown: - yield toppath, dirs, nondirs, topfd + toppath = path.join(toppath, toppath[:0]) # Add trailing slash. + if entries is None: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, None)) + for name in dirs[::-1]) + else: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, entry)) + for name, entry in zip(dirs[::-1], entries[::-1])) __all__.append("fwalk") diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 941fa2b2c5c87f..7dc5784820e847 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1687,8 +1687,6 @@ def test_fd_leak(self): # fwalk() keeps file descriptors open test_walk_many_open_files = None - # fwalk() still uses recursion - test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): diff --git a/Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst b/Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst new file mode 100644 index 00000000000000..92222bc673350f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst @@ -0,0 +1,3 @@ +Fix issue with :func:`os.fwalk` where a :exc:`RecursionError` was raised on +deep directory trees by adjusting the implementation to be iterative instead +of recursive. From a5fef800d31648d19cecc240a2fa0dc71371753e Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 30 May 2024 04:45:47 +0100 Subject: [PATCH 274/903] GH-89727: Fix FD leak on `os.fwalk()` generator finalization. (#119766) Follow-up to 3c890b50. Ensure we `os.close()` open file descriptors when the `os.fwalk()` generator is finalized. --- Lib/os.py | 11 +++++++++-- Lib/test/test_os.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index cef5d90a8c23df..0408e2db79e66e 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -480,8 +480,15 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= top = fspath(top) stack = [(_fwalk_walk, (True, dir_fd, top, top, None))] isbytes = isinstance(top, bytes) - while stack: - yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) + try: + while stack: + yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) + finally: + # Close any file descriptors still on the stack. + while stack: + action, value = stack.pop() + if action == _fwalk_close: + close(value) # Each item in the _fwalk() stack is a pair (action, args). _fwalk_walk = 0 # args: (isroot, dirfd, toppath, topname, entry) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 7dc5784820e847..de5a86f676c4d5 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1685,6 +1685,27 @@ def test_fd_leak(self): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) + @unittest.skipIf( + support.is_emscripten, "Cannot dup stdout on Emscripten" + ) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) + def test_fd_finalization(self): + # Check that close()ing the fwalk() generator closes FDs + def getfd(): + fd = os.dup(1) + os.close(fd) + return fd + for topdown in (False, True): + old_fd = getfd() + it = self.fwalk(os_helper.TESTFN, topdown=topdown) + self.assertEqual(getfd(), old_fd) + next(it) + self.assertGreater(getfd(), old_fd) + it.close() + self.assertEqual(getfd(), old_fd) + # fwalk() keeps file descriptors open test_walk_many_open_files = None From 48f21b3631eb20871fe234e9714b19aa76cf3a49 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 30 May 2024 09:27:32 +0200 Subject: [PATCH 275/903] gh-118235: Move RAISE_SYNTAX_ERROR actions to invalid rules and make sure they stay there (GH-119731) The Full Grammar specification in the docs omits rule actions, so grammar rules that raise a syntax error looked like valid syntax. This was solved in ef940de by hiding those rules in the custom syntax highlighter. This moves all syntax-error alternatives to invalid rules, adds a validator that ensures that actions containing RAISE_SYNTAX_ERROR are in invalid rules, and reverts the syntax highlighter hack. --- Doc/tools/extensions/peg_highlight.py | 2 - Grammar/python.gram | 38 +- .../test_grammar_validator.py | 12 +- Parser/parser.c | 2954 +++++++++-------- Tools/peg_generator/pegen/validator.py | 12 + 5 files changed, 1584 insertions(+), 1434 deletions(-) diff --git a/Doc/tools/extensions/peg_highlight.py b/Doc/tools/extensions/peg_highlight.py index 5ab5530d269901..4bdc2ee1861334 100644 --- a/Doc/tools/extensions/peg_highlight.py +++ b/Doc/tools/extensions/peg_highlight.py @@ -16,7 +16,6 @@ class PEGLexer(RegexLexer): - Rule types - Rule options - Rules named `invalid_*` or `incorrect_*` - - Rules with `RAISE_SYNTAX_ERROR` """ name = "PEG" @@ -60,7 +59,6 @@ class PEGLexer(RegexLexer): (r"^(\s+\|\s+.*invalid_\w+.*\n)", bygroups(None)), (r"^(\s+\|\s+.*incorrect_\w+.*\n)", bygroups(None)), (r"^(#.*invalid syntax.*(?:.|\n)*)", bygroups(None),), - (r"^(\s+\|\s+.*\{[^}]*RAISE_SYNTAX_ERROR[^}]*\})\n", bygroups(None)), ], "root": [ include("invalids"), diff --git a/Grammar/python.gram b/Grammar/python.gram index c04bc641779c04..1734479276dd5b 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -650,17 +650,8 @@ type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ [' type_param[type_param_ty] (memo): | a=NAME b=[type_param_bound] c=[type_param_default] { _PyAST_TypeVar(a->v.Name.id, b, c, EXTRA) } - | '*' a=NAME colon=':' e=expression { - RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind - ? "cannot use constraints with TypeVarTuple" - : "cannot use bound with TypeVarTuple") - } + | invalid_type_param | '*' a=NAME b=[type_param_starred_default] { _PyAST_TypeVarTuple(a->v.Name.id, b, EXTRA) } - | '**' a=NAME colon=':' e=expression { - RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind - ? "cannot use constraints with ParamSpec" - : "cannot use bound with ParamSpec") - } | '**' a=NAME b=[type_param_default] { _PyAST_ParamSpec(a->v.Name.id, b, EXTRA) } type_param_bound[expr_ty]: ':' e=expression { e } @@ -979,8 +970,7 @@ for_if_clause[comprehension_ty]: CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _PyAST_comprehension(a, b, c, 1, p->arena)) } | 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* { _PyAST_comprehension(a, b, c, 0, p->arena) } - | 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' { - RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") } + | invalid_for_if_clause | invalid_for_target listcomp[expr_ty]: @@ -1020,9 +1010,9 @@ kwargs[asdl_seq*]: | ','.kwarg_or_double_starred+ starred_expression[expr_ty]: - | invalid_starred_expression + | invalid_starred_expression_unpacking | '*' a=expression { _PyAST_Starred(a, Load, EXTRA) } - | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") } + | invalid_starred_expression kwarg_or_starred[KeywordOrStarred*]: | invalid_kwarg @@ -1176,6 +1166,18 @@ invalid_legacy_expression: _PyPegen_check_legacy_stmt(p, a) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "Missing parentheses in call to '%U'. Did you mean %U(...)?", a->v.Name.id, a->v.Name.id) : NULL} +invalid_type_param: + | '*' a=NAME colon=':' e=expression { + RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind + ? "cannot use constraints with TypeVarTuple" + : "cannot use bound with TypeVarTuple") + } + | '**' a=NAME colon=':' e=expression { + RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind + ? "cannot use constraints with ParamSpec" + : "cannot use bound with ParamSpec") + } + invalid_expression: # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf" # Soft keywords need to also be ignored because they can be parsed as NAME NAME @@ -1296,6 +1298,10 @@ invalid_with_item: | expression 'as' a=expression &(',' | ')' | ':') { RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } +invalid_for_if_clause: + | 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' { + RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") } + invalid_for_target: | 'async'? 'for' a=star_expressions { RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) } @@ -1409,8 +1415,10 @@ invalid_kvpair: RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") } | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } | expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } -invalid_starred_expression: +invalid_starred_expression_unpacking: | a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") } +invalid_starred_expression: + | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") } invalid_replacement_field: | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") } diff --git a/Lib/test/test_peg_generator/test_grammar_validator.py b/Lib/test/test_peg_generator/test_grammar_validator.py index 72c3d2054fa8f9..c7f20e1de802ce 100644 --- a/Lib/test/test_peg_generator/test_grammar_validator.py +++ b/Lib/test/test_peg_generator/test_grammar_validator.py @@ -4,7 +4,7 @@ test_tools.skip_if_missing("peg_generator") with test_tools.imports_under_tool("peg_generator"): from pegen.grammar_parser import GeneratedParser as GrammarParser - from pegen.validator import SubRuleValidator, ValidationError + from pegen.validator import SubRuleValidator, ValidationError, RaiseRuleValidator from pegen.testutil import parse_string from pegen.grammar import Grammar @@ -49,3 +49,13 @@ def test_rule_with_collision_after_some_other_rules(self) -> None: with self.assertRaises(ValidationError): for rule_name, rule in grammar.rules.items(): validator.validate_rule(rule_name, rule) + + def test_raising_valid_rule(self) -> None: + grammar_source = """ + start: NAME { RAISE_SYNTAX_ERROR("this is not allowed") } + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = RaiseRuleValidator(grammar) + with self.assertRaises(ValidationError): + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) diff --git a/Parser/parser.c b/Parser/parser.c index 7bfc17a92e29de..fec3353d30cb4d 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -29,7 +29,7 @@ static KeywordToken *reserved_keywords[] = { {NULL, -1}, }, (KeywordToken[]) { - {"del", 616}, + {"del", 613}, {"def", 677}, {"for", 672}, {"try", 644}, @@ -43,8 +43,8 @@ static KeywordToken *reserved_keywords[] = { {"with", 635}, {"elif", 664}, {"else", 665}, - {"None", 614}, - {"True", 613}, + {"None", 611}, + {"True", 610}, {NULL, -1}, }, (KeywordToken[]) { @@ -54,7 +54,7 @@ static KeywordToken *reserved_keywords[] = { {"async", 676}, {"class", 679}, {"while", 667}, - {"False", 615}, + {"False", 612}, {"await", 590}, {NULL, -1}, }, @@ -64,7 +64,7 @@ static KeywordToken *reserved_keywords[] = { {"assert", 529}, {"global", 526}, {"except", 657}, - {"lambda", 612}, + {"lambda", 609}, {NULL, -1}, }, (KeywordToken[]) { @@ -283,338 +283,341 @@ static char *soft_keywords[] = { #define invalid_kwarg_type 1196 #define expression_without_invalid_type 1197 #define invalid_legacy_expression_type 1198 -#define invalid_expression_type 1199 -#define invalid_named_expression_type 1200 -#define invalid_assignment_type 1201 -#define invalid_ann_assign_target_type 1202 -#define invalid_del_stmt_type 1203 -#define invalid_block_type 1204 -#define invalid_comprehension_type 1205 -#define invalid_dict_comprehension_type 1206 -#define invalid_parameters_type 1207 -#define invalid_default_type 1208 -#define invalid_star_etc_type 1209 -#define invalid_kwds_type 1210 -#define invalid_parameters_helper_type 1211 -#define invalid_lambda_parameters_type 1212 -#define invalid_lambda_parameters_helper_type 1213 -#define invalid_lambda_star_etc_type 1214 -#define invalid_lambda_kwds_type 1215 -#define invalid_double_type_comments_type 1216 -#define invalid_with_item_type 1217 -#define invalid_for_target_type 1218 -#define invalid_group_type 1219 -#define invalid_import_type 1220 -#define invalid_import_from_targets_type 1221 -#define invalid_compound_stmt_type 1222 -#define invalid_with_stmt_type 1223 -#define invalid_with_stmt_indent_type 1224 -#define invalid_try_stmt_type 1225 -#define invalid_except_stmt_type 1226 -#define invalid_finally_stmt_type 1227 -#define invalid_except_stmt_indent_type 1228 -#define invalid_except_star_stmt_indent_type 1229 -#define invalid_match_stmt_type 1230 -#define invalid_case_block_type 1231 -#define invalid_as_pattern_type 1232 -#define invalid_class_pattern_type 1233 -#define invalid_class_argument_pattern_type 1234 -#define invalid_if_stmt_type 1235 -#define invalid_elif_stmt_type 1236 -#define invalid_else_stmt_type 1237 -#define invalid_while_stmt_type 1238 -#define invalid_for_stmt_type 1239 -#define invalid_def_raw_type 1240 -#define invalid_class_def_raw_type 1241 -#define invalid_double_starred_kvpairs_type 1242 -#define invalid_kvpair_type 1243 -#define invalid_starred_expression_type 1244 -#define invalid_replacement_field_type 1245 -#define invalid_conversion_character_type 1246 -#define invalid_arithmetic_type 1247 -#define invalid_factor_type 1248 -#define invalid_type_params_type 1249 -#define _loop0_1_type 1250 -#define _loop0_2_type 1251 -#define _loop1_3_type 1252 -#define _loop0_5_type 1253 -#define _gather_4_type 1254 -#define _tmp_6_type 1255 -#define _tmp_7_type 1256 -#define _tmp_8_type 1257 -#define _tmp_9_type 1258 -#define _tmp_10_type 1259 -#define _tmp_11_type 1260 -#define _tmp_12_type 1261 -#define _tmp_13_type 1262 -#define _loop1_14_type 1263 -#define _tmp_15_type 1264 -#define _tmp_16_type 1265 -#define _tmp_17_type 1266 -#define _loop0_19_type 1267 -#define _gather_18_type 1268 -#define _loop0_21_type 1269 -#define _gather_20_type 1270 -#define _tmp_22_type 1271 -#define _tmp_23_type 1272 -#define _loop0_24_type 1273 -#define _loop1_25_type 1274 -#define _loop0_27_type 1275 -#define _gather_26_type 1276 -#define _tmp_28_type 1277 -#define _loop0_30_type 1278 -#define _gather_29_type 1279 -#define _tmp_31_type 1280 -#define _loop1_32_type 1281 -#define _tmp_33_type 1282 -#define _tmp_34_type 1283 -#define _tmp_35_type 1284 -#define _loop0_36_type 1285 -#define _loop0_37_type 1286 -#define _loop0_38_type 1287 -#define _loop1_39_type 1288 -#define _loop0_40_type 1289 -#define _loop1_41_type 1290 -#define _loop1_42_type 1291 -#define _loop1_43_type 1292 -#define _loop0_44_type 1293 -#define _loop1_45_type 1294 -#define _loop0_46_type 1295 -#define _loop1_47_type 1296 -#define _loop0_48_type 1297 -#define _loop0_49_type 1298 -#define _loop1_50_type 1299 -#define _loop0_52_type 1300 -#define _gather_51_type 1301 -#define _loop0_54_type 1302 -#define _gather_53_type 1303 -#define _loop0_56_type 1304 -#define _gather_55_type 1305 -#define _loop0_58_type 1306 -#define _gather_57_type 1307 -#define _tmp_59_type 1308 -#define _loop1_60_type 1309 -#define _loop1_61_type 1310 -#define _tmp_62_type 1311 -#define _tmp_63_type 1312 -#define _loop1_64_type 1313 -#define _loop0_66_type 1314 -#define _gather_65_type 1315 -#define _tmp_67_type 1316 -#define _tmp_68_type 1317 -#define _tmp_69_type 1318 -#define _tmp_70_type 1319 -#define _loop0_72_type 1320 -#define _gather_71_type 1321 -#define _loop0_74_type 1322 -#define _gather_73_type 1323 -#define _tmp_75_type 1324 -#define _loop0_77_type 1325 -#define _gather_76_type 1326 -#define _loop0_79_type 1327 -#define _gather_78_type 1328 -#define _loop0_81_type 1329 -#define _gather_80_type 1330 -#define _loop1_82_type 1331 -#define _loop1_83_type 1332 -#define _loop0_85_type 1333 -#define _gather_84_type 1334 -#define _loop1_86_type 1335 -#define _loop1_87_type 1336 -#define _loop1_88_type 1337 -#define _tmp_89_type 1338 -#define _loop0_91_type 1339 -#define _gather_90_type 1340 -#define _tmp_92_type 1341 -#define _tmp_93_type 1342 -#define _tmp_94_type 1343 -#define _tmp_95_type 1344 -#define _tmp_96_type 1345 -#define _tmp_97_type 1346 -#define _loop0_98_type 1347 -#define _loop0_99_type 1348 -#define _loop0_100_type 1349 -#define _loop1_101_type 1350 -#define _loop0_102_type 1351 -#define _loop1_103_type 1352 -#define _loop1_104_type 1353 -#define _loop1_105_type 1354 -#define _loop0_106_type 1355 -#define _loop1_107_type 1356 -#define _loop0_108_type 1357 -#define _loop1_109_type 1358 -#define _loop0_110_type 1359 -#define _loop1_111_type 1360 -#define _loop0_112_type 1361 -#define _loop0_113_type 1362 -#define _loop1_114_type 1363 -#define _tmp_115_type 1364 -#define _loop0_117_type 1365 -#define _gather_116_type 1366 -#define _loop1_118_type 1367 -#define _loop0_119_type 1368 -#define _loop0_120_type 1369 -#define _tmp_121_type 1370 -#define _tmp_122_type 1371 -#define _loop0_124_type 1372 -#define _gather_123_type 1373 -#define _tmp_125_type 1374 -#define _loop0_127_type 1375 -#define _gather_126_type 1376 -#define _loop0_129_type 1377 -#define _gather_128_type 1378 -#define _loop0_131_type 1379 -#define _gather_130_type 1380 -#define _loop0_133_type 1381 -#define _gather_132_type 1382 -#define _loop0_134_type 1383 -#define _loop0_136_type 1384 -#define _gather_135_type 1385 -#define _loop1_137_type 1386 -#define _tmp_138_type 1387 -#define _loop0_140_type 1388 -#define _gather_139_type 1389 -#define _loop0_142_type 1390 -#define _gather_141_type 1391 -#define _loop0_144_type 1392 -#define _gather_143_type 1393 -#define _loop0_146_type 1394 -#define _gather_145_type 1395 -#define _loop0_148_type 1396 -#define _gather_147_type 1397 -#define _tmp_149_type 1398 -#define _tmp_150_type 1399 -#define _loop0_152_type 1400 -#define _gather_151_type 1401 -#define _tmp_153_type 1402 -#define _tmp_154_type 1403 -#define _tmp_155_type 1404 -#define _tmp_156_type 1405 -#define _tmp_157_type 1406 -#define _tmp_158_type 1407 -#define _tmp_159_type 1408 -#define _tmp_160_type 1409 -#define _tmp_161_type 1410 -#define _tmp_162_type 1411 -#define _loop0_163_type 1412 -#define _loop0_164_type 1413 -#define _loop0_165_type 1414 -#define _tmp_166_type 1415 -#define _tmp_167_type 1416 -#define _tmp_168_type 1417 -#define _tmp_169_type 1418 -#define _loop0_170_type 1419 -#define _loop0_171_type 1420 -#define _loop0_172_type 1421 -#define _loop1_173_type 1422 -#define _tmp_174_type 1423 -#define _loop0_175_type 1424 -#define _tmp_176_type 1425 -#define _loop0_177_type 1426 -#define _loop1_178_type 1427 -#define _tmp_179_type 1428 -#define _tmp_180_type 1429 -#define _tmp_181_type 1430 -#define _loop0_182_type 1431 -#define _tmp_183_type 1432 -#define _tmp_184_type 1433 -#define _loop1_185_type 1434 -#define _tmp_186_type 1435 -#define _loop0_187_type 1436 -#define _loop0_188_type 1437 -#define _loop0_189_type 1438 -#define _loop0_191_type 1439 -#define _gather_190_type 1440 -#define _tmp_192_type 1441 -#define _loop0_193_type 1442 -#define _tmp_194_type 1443 -#define _loop0_195_type 1444 -#define _loop1_196_type 1445 -#define _loop1_197_type 1446 -#define _tmp_198_type 1447 -#define _tmp_199_type 1448 -#define _loop0_200_type 1449 -#define _tmp_201_type 1450 -#define _tmp_202_type 1451 -#define _tmp_203_type 1452 -#define _loop0_205_type 1453 -#define _gather_204_type 1454 -#define _loop0_207_type 1455 -#define _gather_206_type 1456 -#define _loop0_209_type 1457 -#define _gather_208_type 1458 -#define _loop0_211_type 1459 -#define _gather_210_type 1460 -#define _loop0_213_type 1461 -#define _gather_212_type 1462 -#define _tmp_214_type 1463 -#define _loop0_215_type 1464 -#define _loop1_216_type 1465 -#define _tmp_217_type 1466 -#define _loop0_218_type 1467 -#define _loop1_219_type 1468 -#define _tmp_220_type 1469 -#define _tmp_221_type 1470 -#define _tmp_222_type 1471 -#define _tmp_223_type 1472 -#define _tmp_224_type 1473 -#define _tmp_225_type 1474 -#define _tmp_226_type 1475 -#define _tmp_227_type 1476 -#define _tmp_228_type 1477 -#define _tmp_229_type 1478 -#define _tmp_230_type 1479 -#define _loop0_232_type 1480 -#define _gather_231_type 1481 -#define _tmp_233_type 1482 -#define _tmp_234_type 1483 -#define _tmp_235_type 1484 -#define _tmp_236_type 1485 -#define _tmp_237_type 1486 -#define _tmp_238_type 1487 -#define _tmp_239_type 1488 -#define _loop0_240_type 1489 -#define _tmp_241_type 1490 -#define _tmp_242_type 1491 -#define _tmp_243_type 1492 -#define _tmp_244_type 1493 -#define _tmp_245_type 1494 -#define _tmp_246_type 1495 -#define _tmp_247_type 1496 -#define _tmp_248_type 1497 -#define _tmp_249_type 1498 -#define _tmp_250_type 1499 -#define _tmp_251_type 1500 -#define _tmp_252_type 1501 -#define _tmp_253_type 1502 -#define _tmp_254_type 1503 -#define _tmp_255_type 1504 -#define _tmp_256_type 1505 -#define _loop0_257_type 1506 -#define _tmp_258_type 1507 -#define _tmp_259_type 1508 -#define _tmp_260_type 1509 -#define _tmp_261_type 1510 -#define _tmp_262_type 1511 -#define _tmp_263_type 1512 -#define _tmp_264_type 1513 -#define _tmp_265_type 1514 -#define _tmp_266_type 1515 -#define _tmp_267_type 1516 -#define _tmp_268_type 1517 -#define _tmp_269_type 1518 -#define _tmp_270_type 1519 -#define _tmp_271_type 1520 -#define _tmp_272_type 1521 -#define _tmp_273_type 1522 -#define _loop0_275_type 1523 -#define _gather_274_type 1524 -#define _tmp_276_type 1525 -#define _tmp_277_type 1526 -#define _tmp_278_type 1527 -#define _tmp_279_type 1528 -#define _tmp_280_type 1529 -#define _tmp_281_type 1530 +#define invalid_type_param_type 1199 +#define invalid_expression_type 1200 +#define invalid_named_expression_type 1201 +#define invalid_assignment_type 1202 +#define invalid_ann_assign_target_type 1203 +#define invalid_del_stmt_type 1204 +#define invalid_block_type 1205 +#define invalid_comprehension_type 1206 +#define invalid_dict_comprehension_type 1207 +#define invalid_parameters_type 1208 +#define invalid_default_type 1209 +#define invalid_star_etc_type 1210 +#define invalid_kwds_type 1211 +#define invalid_parameters_helper_type 1212 +#define invalid_lambda_parameters_type 1213 +#define invalid_lambda_parameters_helper_type 1214 +#define invalid_lambda_star_etc_type 1215 +#define invalid_lambda_kwds_type 1216 +#define invalid_double_type_comments_type 1217 +#define invalid_with_item_type 1218 +#define invalid_for_if_clause_type 1219 +#define invalid_for_target_type 1220 +#define invalid_group_type 1221 +#define invalid_import_type 1222 +#define invalid_import_from_targets_type 1223 +#define invalid_compound_stmt_type 1224 +#define invalid_with_stmt_type 1225 +#define invalid_with_stmt_indent_type 1226 +#define invalid_try_stmt_type 1227 +#define invalid_except_stmt_type 1228 +#define invalid_finally_stmt_type 1229 +#define invalid_except_stmt_indent_type 1230 +#define invalid_except_star_stmt_indent_type 1231 +#define invalid_match_stmt_type 1232 +#define invalid_case_block_type 1233 +#define invalid_as_pattern_type 1234 +#define invalid_class_pattern_type 1235 +#define invalid_class_argument_pattern_type 1236 +#define invalid_if_stmt_type 1237 +#define invalid_elif_stmt_type 1238 +#define invalid_else_stmt_type 1239 +#define invalid_while_stmt_type 1240 +#define invalid_for_stmt_type 1241 +#define invalid_def_raw_type 1242 +#define invalid_class_def_raw_type 1243 +#define invalid_double_starred_kvpairs_type 1244 +#define invalid_kvpair_type 1245 +#define invalid_starred_expression_unpacking_type 1246 +#define invalid_starred_expression_type 1247 +#define invalid_replacement_field_type 1248 +#define invalid_conversion_character_type 1249 +#define invalid_arithmetic_type 1250 +#define invalid_factor_type 1251 +#define invalid_type_params_type 1252 +#define _loop0_1_type 1253 +#define _loop0_2_type 1254 +#define _loop1_3_type 1255 +#define _loop0_5_type 1256 +#define _gather_4_type 1257 +#define _tmp_6_type 1258 +#define _tmp_7_type 1259 +#define _tmp_8_type 1260 +#define _tmp_9_type 1261 +#define _tmp_10_type 1262 +#define _tmp_11_type 1263 +#define _tmp_12_type 1264 +#define _tmp_13_type 1265 +#define _loop1_14_type 1266 +#define _tmp_15_type 1267 +#define _tmp_16_type 1268 +#define _tmp_17_type 1269 +#define _loop0_19_type 1270 +#define _gather_18_type 1271 +#define _loop0_21_type 1272 +#define _gather_20_type 1273 +#define _tmp_22_type 1274 +#define _tmp_23_type 1275 +#define _loop0_24_type 1276 +#define _loop1_25_type 1277 +#define _loop0_27_type 1278 +#define _gather_26_type 1279 +#define _tmp_28_type 1280 +#define _loop0_30_type 1281 +#define _gather_29_type 1282 +#define _tmp_31_type 1283 +#define _loop1_32_type 1284 +#define _tmp_33_type 1285 +#define _tmp_34_type 1286 +#define _tmp_35_type 1287 +#define _loop0_36_type 1288 +#define _loop0_37_type 1289 +#define _loop0_38_type 1290 +#define _loop1_39_type 1291 +#define _loop0_40_type 1292 +#define _loop1_41_type 1293 +#define _loop1_42_type 1294 +#define _loop1_43_type 1295 +#define _loop0_44_type 1296 +#define _loop1_45_type 1297 +#define _loop0_46_type 1298 +#define _loop1_47_type 1299 +#define _loop0_48_type 1300 +#define _loop0_49_type 1301 +#define _loop1_50_type 1302 +#define _loop0_52_type 1303 +#define _gather_51_type 1304 +#define _loop0_54_type 1305 +#define _gather_53_type 1306 +#define _loop0_56_type 1307 +#define _gather_55_type 1308 +#define _loop0_58_type 1309 +#define _gather_57_type 1310 +#define _tmp_59_type 1311 +#define _loop1_60_type 1312 +#define _loop1_61_type 1313 +#define _tmp_62_type 1314 +#define _tmp_63_type 1315 +#define _loop1_64_type 1316 +#define _loop0_66_type 1317 +#define _gather_65_type 1318 +#define _tmp_67_type 1319 +#define _tmp_68_type 1320 +#define _tmp_69_type 1321 +#define _tmp_70_type 1322 +#define _loop0_72_type 1323 +#define _gather_71_type 1324 +#define _loop0_74_type 1325 +#define _gather_73_type 1326 +#define _tmp_75_type 1327 +#define _loop0_77_type 1328 +#define _gather_76_type 1329 +#define _loop0_79_type 1330 +#define _gather_78_type 1331 +#define _loop0_81_type 1332 +#define _gather_80_type 1333 +#define _loop1_82_type 1334 +#define _loop1_83_type 1335 +#define _loop0_85_type 1336 +#define _gather_84_type 1337 +#define _loop1_86_type 1338 +#define _loop1_87_type 1339 +#define _loop1_88_type 1340 +#define _tmp_89_type 1341 +#define _loop0_91_type 1342 +#define _gather_90_type 1343 +#define _tmp_92_type 1344 +#define _tmp_93_type 1345 +#define _tmp_94_type 1346 +#define _tmp_95_type 1347 +#define _tmp_96_type 1348 +#define _tmp_97_type 1349 +#define _loop0_98_type 1350 +#define _loop0_99_type 1351 +#define _loop0_100_type 1352 +#define _loop1_101_type 1353 +#define _loop0_102_type 1354 +#define _loop1_103_type 1355 +#define _loop1_104_type 1356 +#define _loop1_105_type 1357 +#define _loop0_106_type 1358 +#define _loop1_107_type 1359 +#define _loop0_108_type 1360 +#define _loop1_109_type 1361 +#define _loop0_110_type 1362 +#define _loop1_111_type 1363 +#define _loop0_112_type 1364 +#define _loop0_113_type 1365 +#define _loop1_114_type 1366 +#define _tmp_115_type 1367 +#define _loop0_117_type 1368 +#define _gather_116_type 1369 +#define _loop1_118_type 1370 +#define _loop0_119_type 1371 +#define _loop0_120_type 1372 +#define _tmp_121_type 1373 +#define _loop0_123_type 1374 +#define _gather_122_type 1375 +#define _tmp_124_type 1376 +#define _loop0_126_type 1377 +#define _gather_125_type 1378 +#define _loop0_128_type 1379 +#define _gather_127_type 1380 +#define _loop0_130_type 1381 +#define _gather_129_type 1382 +#define _loop0_132_type 1383 +#define _gather_131_type 1384 +#define _loop0_133_type 1385 +#define _loop0_135_type 1386 +#define _gather_134_type 1387 +#define _loop1_136_type 1388 +#define _tmp_137_type 1389 +#define _loop0_139_type 1390 +#define _gather_138_type 1391 +#define _loop0_141_type 1392 +#define _gather_140_type 1393 +#define _loop0_143_type 1394 +#define _gather_142_type 1395 +#define _loop0_145_type 1396 +#define _gather_144_type 1397 +#define _loop0_147_type 1398 +#define _gather_146_type 1399 +#define _tmp_148_type 1400 +#define _tmp_149_type 1401 +#define _loop0_151_type 1402 +#define _gather_150_type 1403 +#define _tmp_152_type 1404 +#define _tmp_153_type 1405 +#define _tmp_154_type 1406 +#define _tmp_155_type 1407 +#define _tmp_156_type 1408 +#define _tmp_157_type 1409 +#define _tmp_158_type 1410 +#define _tmp_159_type 1411 +#define _tmp_160_type 1412 +#define _tmp_161_type 1413 +#define _loop0_162_type 1414 +#define _loop0_163_type 1415 +#define _loop0_164_type 1416 +#define _tmp_165_type 1417 +#define _tmp_166_type 1418 +#define _tmp_167_type 1419 +#define _tmp_168_type 1420 +#define _loop0_169_type 1421 +#define _loop0_170_type 1422 +#define _loop0_171_type 1423 +#define _loop1_172_type 1424 +#define _tmp_173_type 1425 +#define _loop0_174_type 1426 +#define _tmp_175_type 1427 +#define _loop0_176_type 1428 +#define _loop1_177_type 1429 +#define _tmp_178_type 1430 +#define _tmp_179_type 1431 +#define _tmp_180_type 1432 +#define _loop0_181_type 1433 +#define _tmp_182_type 1434 +#define _tmp_183_type 1435 +#define _loop1_184_type 1436 +#define _tmp_185_type 1437 +#define _loop0_186_type 1438 +#define _loop0_187_type 1439 +#define _loop0_188_type 1440 +#define _loop0_190_type 1441 +#define _gather_189_type 1442 +#define _tmp_191_type 1443 +#define _loop0_192_type 1444 +#define _tmp_193_type 1445 +#define _loop0_194_type 1446 +#define _loop1_195_type 1447 +#define _loop1_196_type 1448 +#define _tmp_197_type 1449 +#define _tmp_198_type 1450 +#define _loop0_199_type 1451 +#define _tmp_200_type 1452 +#define _tmp_201_type 1453 +#define _tmp_202_type 1454 +#define _tmp_203_type 1455 +#define _loop0_205_type 1456 +#define _gather_204_type 1457 +#define _loop0_207_type 1458 +#define _gather_206_type 1459 +#define _loop0_209_type 1460 +#define _gather_208_type 1461 +#define _loop0_211_type 1462 +#define _gather_210_type 1463 +#define _loop0_213_type 1464 +#define _gather_212_type 1465 +#define _tmp_214_type 1466 +#define _loop0_215_type 1467 +#define _loop1_216_type 1468 +#define _tmp_217_type 1469 +#define _loop0_218_type 1470 +#define _loop1_219_type 1471 +#define _tmp_220_type 1472 +#define _tmp_221_type 1473 +#define _tmp_222_type 1474 +#define _tmp_223_type 1475 +#define _tmp_224_type 1476 +#define _tmp_225_type 1477 +#define _tmp_226_type 1478 +#define _tmp_227_type 1479 +#define _tmp_228_type 1480 +#define _tmp_229_type 1481 +#define _tmp_230_type 1482 +#define _loop0_232_type 1483 +#define _gather_231_type 1484 +#define _tmp_233_type 1485 +#define _tmp_234_type 1486 +#define _tmp_235_type 1487 +#define _tmp_236_type 1488 +#define _tmp_237_type 1489 +#define _tmp_238_type 1490 +#define _tmp_239_type 1491 +#define _loop0_240_type 1492 +#define _tmp_241_type 1493 +#define _tmp_242_type 1494 +#define _tmp_243_type 1495 +#define _tmp_244_type 1496 +#define _tmp_245_type 1497 +#define _tmp_246_type 1498 +#define _tmp_247_type 1499 +#define _tmp_248_type 1500 +#define _tmp_249_type 1501 +#define _tmp_250_type 1502 +#define _tmp_251_type 1503 +#define _tmp_252_type 1504 +#define _tmp_253_type 1505 +#define _tmp_254_type 1506 +#define _tmp_255_type 1507 +#define _tmp_256_type 1508 +#define _tmp_257_type 1509 +#define _tmp_258_type 1510 +#define _tmp_259_type 1511 +#define _tmp_260_type 1512 +#define _tmp_261_type 1513 +#define _tmp_262_type 1514 +#define _tmp_263_type 1515 +#define _tmp_264_type 1516 +#define _tmp_265_type 1517 +#define _loop0_266_type 1518 +#define _tmp_267_type 1519 +#define _tmp_268_type 1520 +#define _tmp_269_type 1521 +#define _tmp_270_type 1522 +#define _tmp_271_type 1523 +#define _tmp_272_type 1524 +#define _loop0_274_type 1525 +#define _gather_273_type 1526 +#define _tmp_275_type 1527 +#define _tmp_276_type 1528 +#define _tmp_277_type 1529 +#define _tmp_278_type 1530 +#define _tmp_279_type 1531 +#define _tmp_280_type 1532 +#define _tmp_281_type 1533 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -815,6 +818,7 @@ static void *invalid_arguments_rule(Parser *p); static void *invalid_kwarg_rule(Parser *p); static expr_ty expression_without_invalid_rule(Parser *p); static void *invalid_legacy_expression_rule(Parser *p); +static void *invalid_type_param_rule(Parser *p); static void *invalid_expression_rule(Parser *p); static void *invalid_named_expression_rule(Parser *p); static void *invalid_assignment_rule(Parser *p); @@ -834,6 +838,7 @@ static void *invalid_lambda_star_etc_rule(Parser *p); static void *invalid_lambda_kwds_rule(Parser *p); static void *invalid_double_type_comments_rule(Parser *p); static void *invalid_with_item_rule(Parser *p); +static void *invalid_for_if_clause_rule(Parser *p); static void *invalid_for_target_rule(Parser *p); static void *invalid_group_rule(Parser *p); static void *invalid_import_rule(Parser *p); @@ -860,6 +865,7 @@ static void *invalid_def_raw_rule(Parser *p); static void *invalid_class_def_raw_rule(Parser *p); static void *invalid_double_starred_kvpairs_rule(Parser *p); static void *invalid_kvpair_rule(Parser *p); +static void *invalid_starred_expression_unpacking_rule(Parser *p); static void *invalid_starred_expression_rule(Parser *p); static void *invalid_replacement_field_rule(Parser *p); static void *invalid_conversion_character_rule(Parser *p); @@ -987,37 +993,37 @@ static asdl_seq *_loop1_118_rule(Parser *p); static asdl_seq *_loop0_119_rule(Parser *p); static asdl_seq *_loop0_120_rule(Parser *p); static void *_tmp_121_rule(Parser *p); -static void *_tmp_122_rule(Parser *p); -static asdl_seq *_loop0_124_rule(Parser *p); -static asdl_seq *_gather_123_rule(Parser *p); -static void *_tmp_125_rule(Parser *p); -static asdl_seq *_loop0_127_rule(Parser *p); -static asdl_seq *_gather_126_rule(Parser *p); -static asdl_seq *_loop0_129_rule(Parser *p); -static asdl_seq *_gather_128_rule(Parser *p); -static asdl_seq *_loop0_131_rule(Parser *p); -static asdl_seq *_gather_130_rule(Parser *p); +static asdl_seq *_loop0_123_rule(Parser *p); +static asdl_seq *_gather_122_rule(Parser *p); +static void *_tmp_124_rule(Parser *p); +static asdl_seq *_loop0_126_rule(Parser *p); +static asdl_seq *_gather_125_rule(Parser *p); +static asdl_seq *_loop0_128_rule(Parser *p); +static asdl_seq *_gather_127_rule(Parser *p); +static asdl_seq *_loop0_130_rule(Parser *p); +static asdl_seq *_gather_129_rule(Parser *p); +static asdl_seq *_loop0_132_rule(Parser *p); +static asdl_seq *_gather_131_rule(Parser *p); static asdl_seq *_loop0_133_rule(Parser *p); -static asdl_seq *_gather_132_rule(Parser *p); -static asdl_seq *_loop0_134_rule(Parser *p); -static asdl_seq *_loop0_136_rule(Parser *p); -static asdl_seq *_gather_135_rule(Parser *p); -static asdl_seq *_loop1_137_rule(Parser *p); -static void *_tmp_138_rule(Parser *p); -static asdl_seq *_loop0_140_rule(Parser *p); -static asdl_seq *_gather_139_rule(Parser *p); -static asdl_seq *_loop0_142_rule(Parser *p); -static asdl_seq *_gather_141_rule(Parser *p); -static asdl_seq *_loop0_144_rule(Parser *p); -static asdl_seq *_gather_143_rule(Parser *p); -static asdl_seq *_loop0_146_rule(Parser *p); -static asdl_seq *_gather_145_rule(Parser *p); -static asdl_seq *_loop0_148_rule(Parser *p); -static asdl_seq *_gather_147_rule(Parser *p); +static asdl_seq *_loop0_135_rule(Parser *p); +static asdl_seq *_gather_134_rule(Parser *p); +static asdl_seq *_loop1_136_rule(Parser *p); +static void *_tmp_137_rule(Parser *p); +static asdl_seq *_loop0_139_rule(Parser *p); +static asdl_seq *_gather_138_rule(Parser *p); +static asdl_seq *_loop0_141_rule(Parser *p); +static asdl_seq *_gather_140_rule(Parser *p); +static asdl_seq *_loop0_143_rule(Parser *p); +static asdl_seq *_gather_142_rule(Parser *p); +static asdl_seq *_loop0_145_rule(Parser *p); +static asdl_seq *_gather_144_rule(Parser *p); +static asdl_seq *_loop0_147_rule(Parser *p); +static asdl_seq *_gather_146_rule(Parser *p); +static void *_tmp_148_rule(Parser *p); static void *_tmp_149_rule(Parser *p); -static void *_tmp_150_rule(Parser *p); -static asdl_seq *_loop0_152_rule(Parser *p); -static asdl_seq *_gather_151_rule(Parser *p); +static asdl_seq *_loop0_151_rule(Parser *p); +static asdl_seq *_gather_150_rule(Parser *p); +static void *_tmp_152_rule(Parser *p); static void *_tmp_153_rule(Parser *p); static void *_tmp_154_rule(Parser *p); static void *_tmp_155_rule(Parser *p); @@ -1027,45 +1033,45 @@ static void *_tmp_158_rule(Parser *p); static void *_tmp_159_rule(Parser *p); static void *_tmp_160_rule(Parser *p); static void *_tmp_161_rule(Parser *p); -static void *_tmp_162_rule(Parser *p); +static asdl_seq *_loop0_162_rule(Parser *p); static asdl_seq *_loop0_163_rule(Parser *p); static asdl_seq *_loop0_164_rule(Parser *p); -static asdl_seq *_loop0_165_rule(Parser *p); +static void *_tmp_165_rule(Parser *p); static void *_tmp_166_rule(Parser *p); static void *_tmp_167_rule(Parser *p); static void *_tmp_168_rule(Parser *p); -static void *_tmp_169_rule(Parser *p); +static asdl_seq *_loop0_169_rule(Parser *p); static asdl_seq *_loop0_170_rule(Parser *p); static asdl_seq *_loop0_171_rule(Parser *p); -static asdl_seq *_loop0_172_rule(Parser *p); -static asdl_seq *_loop1_173_rule(Parser *p); -static void *_tmp_174_rule(Parser *p); -static asdl_seq *_loop0_175_rule(Parser *p); -static void *_tmp_176_rule(Parser *p); -static asdl_seq *_loop0_177_rule(Parser *p); -static asdl_seq *_loop1_178_rule(Parser *p); +static asdl_seq *_loop1_172_rule(Parser *p); +static void *_tmp_173_rule(Parser *p); +static asdl_seq *_loop0_174_rule(Parser *p); +static void *_tmp_175_rule(Parser *p); +static asdl_seq *_loop0_176_rule(Parser *p); +static asdl_seq *_loop1_177_rule(Parser *p); +static void *_tmp_178_rule(Parser *p); static void *_tmp_179_rule(Parser *p); static void *_tmp_180_rule(Parser *p); -static void *_tmp_181_rule(Parser *p); -static asdl_seq *_loop0_182_rule(Parser *p); +static asdl_seq *_loop0_181_rule(Parser *p); +static void *_tmp_182_rule(Parser *p); static void *_tmp_183_rule(Parser *p); -static void *_tmp_184_rule(Parser *p); -static asdl_seq *_loop1_185_rule(Parser *p); -static void *_tmp_186_rule(Parser *p); +static asdl_seq *_loop1_184_rule(Parser *p); +static void *_tmp_185_rule(Parser *p); +static asdl_seq *_loop0_186_rule(Parser *p); static asdl_seq *_loop0_187_rule(Parser *p); static asdl_seq *_loop0_188_rule(Parser *p); -static asdl_seq *_loop0_189_rule(Parser *p); -static asdl_seq *_loop0_191_rule(Parser *p); -static asdl_seq *_gather_190_rule(Parser *p); -static void *_tmp_192_rule(Parser *p); -static asdl_seq *_loop0_193_rule(Parser *p); -static void *_tmp_194_rule(Parser *p); -static asdl_seq *_loop0_195_rule(Parser *p); +static asdl_seq *_loop0_190_rule(Parser *p); +static asdl_seq *_gather_189_rule(Parser *p); +static void *_tmp_191_rule(Parser *p); +static asdl_seq *_loop0_192_rule(Parser *p); +static void *_tmp_193_rule(Parser *p); +static asdl_seq *_loop0_194_rule(Parser *p); +static asdl_seq *_loop1_195_rule(Parser *p); static asdl_seq *_loop1_196_rule(Parser *p); -static asdl_seq *_loop1_197_rule(Parser *p); +static void *_tmp_197_rule(Parser *p); static void *_tmp_198_rule(Parser *p); -static void *_tmp_199_rule(Parser *p); -static asdl_seq *_loop0_200_rule(Parser *p); +static asdl_seq *_loop0_199_rule(Parser *p); +static void *_tmp_200_rule(Parser *p); static void *_tmp_201_rule(Parser *p); static void *_tmp_202_rule(Parser *p); static void *_tmp_203_rule(Parser *p); @@ -1122,7 +1128,7 @@ static void *_tmp_253_rule(Parser *p); static void *_tmp_254_rule(Parser *p); static void *_tmp_255_rule(Parser *p); static void *_tmp_256_rule(Parser *p); -static asdl_seq *_loop0_257_rule(Parser *p); +static void *_tmp_257_rule(Parser *p); static void *_tmp_258_rule(Parser *p); static void *_tmp_259_rule(Parser *p); static void *_tmp_260_rule(Parser *p); @@ -1131,16 +1137,16 @@ static void *_tmp_262_rule(Parser *p); static void *_tmp_263_rule(Parser *p); static void *_tmp_264_rule(Parser *p); static void *_tmp_265_rule(Parser *p); -static void *_tmp_266_rule(Parser *p); +static asdl_seq *_loop0_266_rule(Parser *p); static void *_tmp_267_rule(Parser *p); static void *_tmp_268_rule(Parser *p); static void *_tmp_269_rule(Parser *p); static void *_tmp_270_rule(Parser *p); static void *_tmp_271_rule(Parser *p); static void *_tmp_272_rule(Parser *p); -static void *_tmp_273_rule(Parser *p); -static asdl_seq *_loop0_275_rule(Parser *p); -static asdl_seq *_gather_274_rule(Parser *p); +static asdl_seq *_loop0_274_rule(Parser *p); +static asdl_seq *_gather_273_rule(Parser *p); +static void *_tmp_275_rule(Parser *p); static void *_tmp_276_rule(Parser *p); static void *_tmp_277_rule(Parser *p); static void *_tmp_278_rule(Parser *p); @@ -1885,7 +1891,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'del' del_stmt")); stmt_ty del_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 616) // token='del' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 613) // token='del' && (del_stmt_var = del_stmt_rule(p)) // del_stmt ) @@ -3215,7 +3221,7 @@ del_stmt_rule(Parser *p) Token * _keyword; asdl_expr_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 616)) // token='del' + (_keyword = _PyPegen_expect_token(p, 613)) // token='del' && (a = del_targets_rule(p)) // del_targets && @@ -8328,7 +8334,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 614)) // token='None' + (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -8361,7 +8367,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 613)) // token='True' + (_keyword = _PyPegen_expect_token(p, 610)) // token='True' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -8394,7 +8400,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 615)) // token='False' + (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -8520,7 +8526,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 614)) // token='None' + (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -8553,7 +8559,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 613)) // token='True' + (_keyword = _PyPegen_expect_token(p, 610)) // token='True' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -8586,7 +8592,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 615)) // token='False' + (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -10674,9 +10680,8 @@ type_param_seq_rule(Parser *p) // type_param: // | NAME type_param_bound? type_param_default? -// | '*' NAME ':' expression +// | invalid_type_param // | '*' NAME type_param_starred_default? -// | '**' NAME ':' expression // | '**' NAME type_param_default? static type_param_ty type_param_rule(Parser *p) @@ -10742,38 +10747,24 @@ type_param_rule(Parser *p) D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME type_param_bound? type_param_default?")); } - { // '*' NAME ':' expression + if (p->call_invalid_rules) { // invalid_type_param if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' NAME ':' expression")); - Token * _literal; - expr_ty a; - Token * colon; - expr_ty e; + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_type_param")); + void *invalid_type_param_var; if ( - (_literal = _PyPegen_expect_token(p, 16)) // token='*' - && - (a = _PyPegen_name_token(p)) // NAME - && - (colon = _PyPegen_expect_token(p, 11)) // token=':' - && - (e = expression_rule(p)) // expression + (invalid_type_param_var = invalid_type_param_rule(p)) // invalid_type_param ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME ':' expression")); - _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( colon , e -> kind == Tuple_kind ? "cannot use constraints with TypeVarTuple" : "cannot use bound with TypeVarTuple" ); - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_type_param")); + _res = invalid_type_param_var; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME ':' expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_type_param")); } { // '*' NAME type_param_starred_default? if (p->error_indicator) { @@ -10814,39 +10805,6 @@ type_param_rule(Parser *p) D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME type_param_starred_default?")); } - { // '**' NAME ':' expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' NAME ':' expression")); - Token * _literal; - expr_ty a; - Token * colon; - expr_ty e; - if ( - (_literal = _PyPegen_expect_token(p, 35)) // token='**' - && - (a = _PyPegen_name_token(p)) // NAME - && - (colon = _PyPegen_expect_token(p, 11)) // token=':' - && - (e = expression_rule(p)) // expression - ) - { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME ':' expression")); - _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( colon , e -> kind == Tuple_kind ? "cannot use constraints with ParamSpec" : "cannot use bound with ParamSpec" ); - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME ':' expression")); - } { // '**' NAME type_param_default? if (p->error_indicator) { p->level--; @@ -14780,7 +14738,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 613)) // token='True' + (_keyword = _PyPegen_expect_token(p, 610)) // token='True' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -14813,7 +14771,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 615)) // token='False' + (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -14846,7 +14804,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 614)) // token='None' + (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -15114,7 +15072,7 @@ lambdef_rule(Parser *p) void *a; expr_ty b; if ( - (_keyword = _PyPegen_expect_token(p, 612)) // token='lambda' + (_keyword = _PyPegen_expect_token(p, 609)) // token='lambda' && (a = lambda_params_rule(p), !p->error_indicator) // lambda_params? && @@ -17017,7 +16975,7 @@ for_if_clauses_rule(Parser *p) // for_if_clause: // | 'async' 'for' star_targets 'in' ~ disjunction (('if' disjunction))* // | 'for' star_targets 'in' ~ disjunction (('if' disjunction))* -// | 'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in' +// | invalid_for_if_clause // | invalid_for_target static comprehension_ty for_if_clause_rule(Parser *p) @@ -17120,38 +17078,24 @@ for_if_clause_rule(Parser *p) return NULL; } } - { // 'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in' + if (p->call_invalid_rules) { // invalid_for_if_clause if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> for_if_clause[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); - Token * _keyword; - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_121_var; + D(fprintf(stderr, "%*c> for_if_clause[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_for_if_clause")); + void *invalid_for_if_clause_var; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? - && - (_keyword = _PyPegen_expect_token(p, 672)) // token='for' - && - (_tmp_121_var = _tmp_121_rule(p)) // bitwise_or ((',' bitwise_or))* ','? - && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 673) // token='in' + (invalid_for_if_clause_var = invalid_for_if_clause_rule(p)) // invalid_for_if_clause ) { - D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); - _res = RAISE_SYNTAX_ERROR ( "'in' expected after for-loop variables" ); - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_for_if_clause")); + _res = invalid_for_if_clause_var; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s for_if_clause[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_for_if_clause")); } if (p->call_invalid_rules) { // invalid_for_target if (p->error_indicator) { @@ -17393,7 +17337,7 @@ genexp_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_122_rule(p)) // assignment_expression | expression !':=' + (a = _tmp_121_rule(p)) // assignment_expression | expression !':=' && (b = for_if_clauses_rule(p)) // for_if_clauses && @@ -17642,9 +17586,9 @@ args_rule(Parser *p) asdl_expr_seq* a; void *b; if ( - (a = (asdl_expr_seq*)_gather_123_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (a = (asdl_expr_seq*)_gather_122_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && - (b = _tmp_125_rule(p), !p->error_indicator) // [',' kwargs] + (b = _tmp_124_rule(p), !p->error_indicator) // [',' kwargs] ) { D(fprintf(stderr, "%*c+ args[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ [',' kwargs]")); @@ -17734,11 +17678,11 @@ kwargs_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _gather_126_rule(p)) // ','.kwarg_or_starred+ + (a = _gather_125_rule(p)) // ','.kwarg_or_starred+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _gather_128_rule(p)) // ','.kwarg_or_double_starred+ + (b = _gather_127_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+")); @@ -17760,13 +17704,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - asdl_seq * _gather_130_var; + asdl_seq * _gather_129_var; if ( - (_gather_130_var = _gather_130_rule(p)) // ','.kwarg_or_starred+ + (_gather_129_var = _gather_129_rule(p)) // ','.kwarg_or_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - _res = _gather_130_var; + _res = _gather_129_var; goto done; } p->mark = _mark; @@ -17779,13 +17723,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - asdl_seq * _gather_132_var; + asdl_seq * _gather_131_var; if ( - (_gather_132_var = _gather_132_rule(p)) // ','.kwarg_or_double_starred+ + (_gather_131_var = _gather_131_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - _res = _gather_132_var; + _res = _gather_131_var; goto done; } p->mark = _mark; @@ -17798,7 +17742,10 @@ kwargs_rule(Parser *p) return _res; } -// starred_expression: invalid_starred_expression | '*' expression | '*' +// starred_expression: +// | invalid_starred_expression_unpacking +// | '*' expression +// | invalid_starred_expression static expr_ty starred_expression_rule(Parser *p) { @@ -17820,24 +17767,24 @@ starred_expression_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - if (p->call_invalid_rules) { // invalid_starred_expression + if (p->call_invalid_rules) { // invalid_starred_expression_unpacking if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression")); - void *invalid_starred_expression_var; + D(fprintf(stderr, "%*c> starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression_unpacking")); + void *invalid_starred_expression_unpacking_var; if ( - (invalid_starred_expression_var = invalid_starred_expression_rule(p)) // invalid_starred_expression + (invalid_starred_expression_unpacking_var = invalid_starred_expression_unpacking_rule(p)) // invalid_starred_expression_unpacking ) { - D(fprintf(stderr, "%*c+ starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression")); - _res = invalid_starred_expression_var; + D(fprintf(stderr, "%*c+ starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression_unpacking")); + _res = invalid_starred_expression_unpacking_var; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s starred_expression[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_starred_expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_starred_expression_unpacking")); } { // '*' expression if (p->error_indicator) { @@ -17875,29 +17822,24 @@ starred_expression_rule(Parser *p) D(fprintf(stderr, "%*c%s starred_expression[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' expression")); } - { // '*' + if (p->call_invalid_rules) { // invalid_starred_expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); - Token * _literal; + D(fprintf(stderr, "%*c> starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression")); + void *invalid_starred_expression_var; if ( - (_literal = _PyPegen_expect_token(p, 16)) // token='*' + (invalid_starred_expression_var = invalid_starred_expression_rule(p)) // invalid_starred_expression ) { - D(fprintf(stderr, "%*c+ starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); - _res = RAISE_SYNTAX_ERROR ( "Invalid star expression" ); - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } + D(fprintf(stderr, "%*c+ starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_starred_expression")); + _res = invalid_starred_expression_var; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s starred_expression[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_starred_expression")); } _res = NULL; done: @@ -18198,7 +18140,7 @@ star_targets_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop0_134_rule(p)) // ((',' star_target))* + (b = _loop0_133_rule(p)) // ((',' star_target))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18254,7 +18196,7 @@ star_targets_list_seq_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_135_rule(p)) // ','.star_target+ + (a = (asdl_expr_seq*)_gather_134_rule(p)) // ','.star_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18304,7 +18246,7 @@ star_targets_tuple_seq_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop1_137_rule(p)) // ((',' star_target))+ + (b = _loop1_136_rule(p)) // ((',' star_target))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18392,7 +18334,7 @@ star_target_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (a = _tmp_138_rule(p)) // !'*' star_target + (a = _tmp_137_rule(p)) // !'*' star_target ) { D(fprintf(stderr, "%*c+ star_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (!'*' star_target)")); @@ -19315,7 +19257,7 @@ del_targets_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_139_rule(p)) // ','.del_target+ + (a = (asdl_expr_seq*)_gather_138_rule(p)) // ','.del_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -19673,7 +19615,7 @@ type_expressions_rule(Parser *p) expr_ty b; expr_ty c; if ( - (a = _gather_141_rule(p)) // ','.expression+ + (a = _gather_140_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19712,7 +19654,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_143_rule(p)) // ','.expression+ + (a = _gather_142_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19745,7 +19687,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_145_rule(p)) // ','.expression+ + (a = _gather_144_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -19865,7 +19807,7 @@ type_expressions_rule(Parser *p) D(fprintf(stderr, "%*c> type_expressions[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.expression+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_147_rule(p)) // ','.expression+ + (a = (asdl_expr_seq*)_gather_146_rule(p)) // ','.expression+ ) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.expression+")); @@ -19916,7 +19858,7 @@ func_type_comment_rule(Parser *p) && (t = _PyPegen_expect_token(p, TYPE_COMMENT)) // token='TYPE_COMMENT' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_149_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_148_rule, p) ) { D(fprintf(stderr, "%*c+ func_type_comment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE TYPE_COMMENT &(NEWLINE INDENT)")); @@ -20002,15 +19944,15 @@ invalid_arguments_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); - asdl_seq * _gather_151_var; - void *_tmp_150_var; + asdl_seq * _gather_150_var; + void *_tmp_149_var; Token * a; if ( - (_tmp_150_var = _tmp_150_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs + (_tmp_149_var = _tmp_149_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs && (a = _PyPegen_expect_token(p, 12)) // token=',' && - (_gather_151_var = _gather_151_rule(p)) // ','.(starred_expression !'=')+ + (_gather_150_var = _gather_150_rule(p)) // ','.(starred_expression !'=')+ ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); @@ -20044,7 +19986,7 @@ invalid_arguments_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_153_rule(p), !p->error_indicator) // [args | expression for_if_clauses] + (_opt_var = _tmp_152_rule(p), !p->error_indicator) // [args | expression for_if_clauses] ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); @@ -20104,13 +20046,13 @@ invalid_arguments_rule(Parser *p) expr_ty a; Token * b; if ( - (_opt_var = _tmp_154_rule(p), !p->error_indicator) // [(args ',')] + (_opt_var = _tmp_153_rule(p), !p->error_indicator) // [(args ',')] && (a = _PyPegen_name_token(p)) // NAME && (b = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_155_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_154_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(args ',')] NAME '=' &(',' | ')')")); @@ -20248,7 +20190,7 @@ invalid_kwarg_rule(Parser *p) Token* a; Token * b; if ( - (a = (Token*)_tmp_156_rule(p)) // 'True' | 'False' | 'None' + (a = (Token*)_tmp_155_rule(p)) // 'True' | 'False' | 'None' && (b = _PyPegen_expect_token(p, 22)) // token='=' ) @@ -20308,7 +20250,7 @@ invalid_kwarg_rule(Parser *p) expr_ty a; Token * b; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_157_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_156_rule, p) && (a = expression_rule(p)) // expression && @@ -20539,6 +20481,91 @@ invalid_legacy_expression_rule(Parser *p) return _res; } +// invalid_type_param: '*' NAME ':' expression | '**' NAME ':' expression +static void * +invalid_type_param_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '*' NAME ':' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' NAME ':' expression")); + Token * _literal; + expr_ty a; + Token * colon; + expr_ty e; + if ( + (_literal = _PyPegen_expect_token(p, 16)) // token='*' + && + (a = _PyPegen_name_token(p)) // NAME + && + (colon = _PyPegen_expect_token(p, 11)) // token=':' + && + (e = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ invalid_type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME ':' expression")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( colon , e -> kind == Tuple_kind ? "cannot use constraints with TypeVarTuple" : "cannot use bound with TypeVarTuple" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_type_param[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME ':' expression")); + } + { // '**' NAME ':' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' NAME ':' expression")); + Token * _literal; + expr_ty a; + Token * colon; + expr_ty e; + if ( + (_literal = _PyPegen_expect_token(p, 35)) // token='**' + && + (a = _PyPegen_name_token(p)) // NAME + && + (colon = _PyPegen_expect_token(p, 11)) // token=':' + && + (e = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ invalid_type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME ':' expression")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( colon , e -> kind == Tuple_kind ? "cannot use constraints with ParamSpec" : "cannot use bound with ParamSpec" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_type_param[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME ':' expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_expression: // | !(NAME STRING | SOFT_KEYWORD) disjunction expression_without_invalid // | disjunction 'if' disjunction !('else' | ':') @@ -20564,7 +20591,7 @@ invalid_expression_rule(Parser *p) expr_ty a; expr_ty b; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_158_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_157_rule, p) && (a = disjunction_rule(p)) // disjunction && @@ -20600,7 +20627,7 @@ invalid_expression_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_159_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_158_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); @@ -20627,7 +20654,7 @@ invalid_expression_rule(Parser *p) Token * a; Token * b; if ( - (a = _PyPegen_expect_token(p, 612)) // token='lambda' + (a = _PyPegen_expect_token(p, 609)) // token='lambda' && (_opt_var = lambda_params_rule(p), !p->error_indicator) // lambda_params? && @@ -20721,7 +20748,7 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_160_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_159_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); @@ -20747,7 +20774,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_161_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_160_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -20755,7 +20782,7 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_162_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_161_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); @@ -20835,7 +20862,7 @@ invalid_assignment_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions* ':' expression")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_163_var; + asdl_seq * _loop0_162_var; expr_ty a; expr_ty expression_var; if ( @@ -20843,7 +20870,7 @@ invalid_assignment_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_loop0_163_var = _loop0_163_rule(p)) // star_named_expressions* + (_loop0_162_var = _loop0_162_rule(p)) // star_named_expressions* && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -20900,10 +20927,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* star_expressions '='")); Token * _literal; - asdl_seq * _loop0_164_var; + asdl_seq * _loop0_163_var; expr_ty a; if ( - (_loop0_164_var = _loop0_164_rule(p)) // ((star_targets '='))* + (_loop0_163_var = _loop0_163_rule(p)) // ((star_targets '='))* && (a = star_expressions_rule(p)) // star_expressions && @@ -20930,10 +20957,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* yield_expr '='")); Token * _literal; - asdl_seq * _loop0_165_var; + asdl_seq * _loop0_164_var; expr_ty a; if ( - (_loop0_165_var = _loop0_165_rule(p)) // ((star_targets '='))* + (_loop0_164_var = _loop0_164_rule(p)) // ((star_targets '='))* && (a = yield_expr_rule(p)) // yield_expr && @@ -21098,7 +21125,7 @@ invalid_del_stmt_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 616)) // token='del' + (_keyword = _PyPegen_expect_token(p, 613)) // token='del' && (a = star_expressions_rule(p)) // star_expressions ) @@ -21189,11 +21216,11 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '(' | '{') starred_expression for_if_clauses")); - void *_tmp_166_var; + void *_tmp_165_var; expr_ty a; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_166_var = _tmp_166_rule(p)) // '[' | '(' | '{' + (_tmp_165_var = _tmp_165_rule(p)) // '[' | '(' | '{' && (a = starred_expression_rule(p)) // starred_expression && @@ -21220,12 +21247,12 @@ invalid_comprehension_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' star_named_expressions for_if_clauses")); Token * _literal; - void *_tmp_167_var; + void *_tmp_166_var; expr_ty a; asdl_expr_seq* b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_167_var = _tmp_167_rule(p)) // '[' | '{' + (_tmp_166_var = _tmp_166_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -21255,12 +21282,12 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' for_if_clauses")); - void *_tmp_168_var; + void *_tmp_167_var; expr_ty a; Token * b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_168_var = _tmp_168_rule(p)) // '[' | '{' + (_tmp_167_var = _tmp_167_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -21395,13 +21422,13 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); - asdl_seq * _loop0_170_var; - void *_tmp_169_var; + asdl_seq * _loop0_169_var; + void *_tmp_168_var; Token * a; if ( - (_tmp_169_var = _tmp_169_rule(p)) // slash_no_default | slash_with_default + (_tmp_168_var = _tmp_168_rule(p)) // slash_no_default | slash_with_default && - (_loop0_170_var = _loop0_170_rule(p)) // param_maybe_default* + (_loop0_169_var = _loop0_169_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21425,7 +21452,7 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default? param_no_default* invalid_parameters_helper param_no_default")); - asdl_seq * _loop0_171_var; + asdl_seq * _loop0_170_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -21433,7 +21460,7 @@ invalid_parameters_rule(Parser *p) if ( (_opt_var = slash_no_default_rule(p), !p->error_indicator) // slash_no_default? && - (_loop0_171_var = _loop0_171_rule(p)) // param_no_default* + (_loop0_170_var = _loop0_170_rule(p)) // param_no_default* && (invalid_parameters_helper_var = invalid_parameters_helper_rule(p)) // invalid_parameters_helper && @@ -21459,18 +21486,18 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default* '(' param_no_default+ ','? ')'")); - asdl_seq * _loop0_172_var; - asdl_seq * _loop1_173_var; + asdl_seq * _loop0_171_var; + asdl_seq * _loop1_172_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_172_var = _loop0_172_rule(p)) // param_no_default* + (_loop0_171_var = _loop0_171_rule(p)) // param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_loop1_173_var = _loop1_173_rule(p)) // param_no_default+ + (_loop1_172_var = _loop1_172_rule(p)) // param_no_default+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -21497,22 +21524,22 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(slash_no_default | slash_with_default)] param_maybe_default* '*' (',' | param_no_default) param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_175_var; - asdl_seq * _loop0_177_var; + asdl_seq * _loop0_174_var; + asdl_seq * _loop0_176_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_176_var; + void *_tmp_175_var; Token * a; if ( - (_opt_var = _tmp_174_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] + (_opt_var = _tmp_173_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] && - (_loop0_175_var = _loop0_175_rule(p)) // param_maybe_default* + (_loop0_174_var = _loop0_174_rule(p)) // param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_176_var = _tmp_176_rule(p)) // ',' | param_no_default + (_tmp_175_var = _tmp_175_rule(p)) // ',' | param_no_default && - (_loop0_177_var = _loop0_177_rule(p)) // param_maybe_default* + (_loop0_176_var = _loop0_176_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -21537,10 +21564,10 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_178_var; + asdl_seq * _loop1_177_var; Token * a; if ( - (_loop1_178_var = _loop1_178_rule(p)) // param_maybe_default+ + (_loop1_177_var = _loop1_177_rule(p)) // param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -21589,7 +21616,7 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_179_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_178_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); @@ -21634,12 +21661,12 @@ invalid_star_etc_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); - void *_tmp_180_var; + void *_tmp_179_var; Token * a; if ( (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_180_var = _tmp_180_rule(p)) // ')' | ',' (')' | '**') + (_tmp_179_var = _tmp_179_rule(p)) // ')' | ',' (')' | '**') ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); @@ -21722,20 +21749,20 @@ invalid_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_182_var; - void *_tmp_181_var; - void *_tmp_183_var; + asdl_seq * _loop0_181_var; + void *_tmp_180_var; + void *_tmp_182_var; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_181_var = _tmp_181_rule(p)) // param_no_default | ',' + (_tmp_180_var = _tmp_180_rule(p)) // param_no_default | ',' && - (_loop0_182_var = _loop0_182_rule(p)) // param_maybe_default* + (_loop0_181_var = _loop0_181_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_183_var = _tmp_183_rule(p)) // param_no_default | ',' + (_tmp_182_var = _tmp_182_rule(p)) // param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); @@ -21850,7 +21877,7 @@ invalid_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_184_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_183_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' ('*' | '**' | '/')")); @@ -21915,13 +21942,13 @@ invalid_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - asdl_seq * _loop1_185_var; + asdl_seq * _loop1_184_var; if ( - (_loop1_185_var = _loop1_185_rule(p)) // param_with_default+ + (_loop1_184_var = _loop1_184_rule(p)) // param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - _res = _loop1_185_var; + _res = _loop1_184_var; goto done; } p->mark = _mark; @@ -21986,13 +22013,13 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); - asdl_seq * _loop0_187_var; - void *_tmp_186_var; + asdl_seq * _loop0_186_var; + void *_tmp_185_var; Token * a; if ( - (_tmp_186_var = _tmp_186_rule(p)) // lambda_slash_no_default | lambda_slash_with_default + (_tmp_185_var = _tmp_185_rule(p)) // lambda_slash_no_default | lambda_slash_with_default && - (_loop0_187_var = _loop0_187_rule(p)) // lambda_param_maybe_default* + (_loop0_186_var = _loop0_186_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22016,7 +22043,7 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper lambda_param_no_default")); - asdl_seq * _loop0_188_var; + asdl_seq * _loop0_187_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -22024,7 +22051,7 @@ invalid_lambda_parameters_rule(Parser *p) if ( (_opt_var = lambda_slash_no_default_rule(p), !p->error_indicator) // lambda_slash_no_default? && - (_loop0_188_var = _loop0_188_rule(p)) // lambda_param_no_default* + (_loop0_187_var = _loop0_187_rule(p)) // lambda_param_no_default* && (invalid_lambda_parameters_helper_var = invalid_lambda_parameters_helper_rule(p)) // invalid_lambda_parameters_helper && @@ -22050,18 +22077,18 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); - asdl_seq * _gather_190_var; - asdl_seq * _loop0_189_var; + asdl_seq * _gather_189_var; + asdl_seq * _loop0_188_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_189_var = _loop0_189_rule(p)) // lambda_param_no_default* + (_loop0_188_var = _loop0_188_rule(p)) // lambda_param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_190_var = _gather_190_rule(p)) // ','.lambda_param+ + (_gather_189_var = _gather_189_rule(p)) // ','.lambda_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22088,22 +22115,22 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(lambda_slash_no_default | lambda_slash_with_default)] lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_193_var; - asdl_seq * _loop0_195_var; + asdl_seq * _loop0_192_var; + asdl_seq * _loop0_194_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_194_var; + void *_tmp_193_var; Token * a; if ( - (_opt_var = _tmp_192_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] + (_opt_var = _tmp_191_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] && - (_loop0_193_var = _loop0_193_rule(p)) // lambda_param_maybe_default* + (_loop0_192_var = _loop0_192_rule(p)) // lambda_param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_194_var = _tmp_194_rule(p)) // ',' | lambda_param_no_default + (_tmp_193_var = _tmp_193_rule(p)) // ',' | lambda_param_no_default && - (_loop0_195_var = _loop0_195_rule(p)) // lambda_param_maybe_default* + (_loop0_194_var = _loop0_194_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22128,10 +22155,10 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_196_var; + asdl_seq * _loop1_195_var; Token * a; if ( - (_loop1_196_var = _loop1_196_rule(p)) // lambda_param_maybe_default+ + (_loop1_195_var = _loop1_195_rule(p)) // lambda_param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -22202,13 +22229,13 @@ invalid_lambda_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - asdl_seq * _loop1_197_var; + asdl_seq * _loop1_196_var; if ( - (_loop1_197_var = _loop1_197_rule(p)) // lambda_param_with_default+ + (_loop1_196_var = _loop1_196_rule(p)) // lambda_param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - _res = _loop1_197_var; + _res = _loop1_196_var; goto done; } p->mark = _mark; @@ -22244,11 +22271,11 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); Token * _literal; - void *_tmp_198_var; + void *_tmp_197_var; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_198_var = _tmp_198_rule(p)) // ':' | ',' (':' | '**') + (_tmp_197_var = _tmp_197_rule(p)) // ':' | ',' (':' | '**') ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); @@ -22301,20 +22328,20 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_200_var; - void *_tmp_199_var; - void *_tmp_201_var; + asdl_seq * _loop0_199_var; + void *_tmp_198_var; + void *_tmp_200_var; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_199_var = _tmp_199_rule(p)) // lambda_param_no_default | ',' + (_tmp_198_var = _tmp_198_rule(p)) // lambda_param_no_default | ',' && - (_loop0_200_var = _loop0_200_rule(p)) // lambda_param_maybe_default* + (_loop0_199_var = _loop0_199_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_201_var = _tmp_201_rule(p)) // lambda_param_no_default | ',' + (_tmp_200_var = _tmp_200_rule(p)) // lambda_param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); @@ -22432,7 +22459,7 @@ invalid_lambda_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_202_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_201_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' ('*' | '**' | '/')")); @@ -22538,7 +22565,7 @@ invalid_with_item_rule(Parser *p) && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_203_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_202_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); @@ -22560,6 +22587,58 @@ invalid_with_item_rule(Parser *p) return _res; } +// invalid_for_if_clause: 'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in' +static void * +invalid_for_if_clause_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // 'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_for_if_clause[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); + Token * _keyword; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + void *_tmp_203_var; + if ( + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + && + (_keyword = _PyPegen_expect_token(p, 672)) // token='for' + && + (_tmp_203_var = _tmp_203_rule(p)) // bitwise_or ((',' bitwise_or))* ','? + && + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 673) // token='in' + ) + { + D(fprintf(stderr, "%*c+ invalid_for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); + _res = RAISE_SYNTAX_ERROR ( "'in' expected after for-loop variables" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_for_if_clause[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_for_target: 'async'? 'for' star_expressions static void * invalid_for_target_rule(Parser *p) @@ -24906,9 +24985,9 @@ invalid_kvpair_rule(Parser *p) return _res; } -// invalid_starred_expression: '*' expression '=' expression +// invalid_starred_expression_unpacking: '*' expression '=' expression static void * -invalid_starred_expression_rule(Parser *p) +invalid_starred_expression_unpacking_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -24924,7 +25003,7 @@ invalid_starred_expression_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' expression '=' expression")); + D(fprintf(stderr, "%*c> invalid_starred_expression_unpacking[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' expression '=' expression")); Token * _literal; Token * a; expr_ty b; @@ -24939,7 +25018,7 @@ invalid_starred_expression_rule(Parser *p) (b = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ invalid_starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' expression '=' expression")); + D(fprintf(stderr, "%*c+ invalid_starred_expression_unpacking[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' expression '=' expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot assign to iterable argument unpacking" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24949,7 +25028,7 @@ invalid_starred_expression_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s invalid_starred_expression[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s invalid_starred_expression_unpacking[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' expression '=' expression")); } _res = NULL; @@ -24958,6 +25037,49 @@ invalid_starred_expression_rule(Parser *p) return _res; } +// invalid_starred_expression: '*' +static void * +invalid_starred_expression_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '*' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_starred_expression[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 16)) // token='*' + ) + { + D(fprintf(stderr, "%*c+ invalid_starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + _res = RAISE_SYNTAX_ERROR ( "Invalid star expression" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_starred_expression[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_replacement_field: // | '{' '=' // | '{' '!' @@ -33074,54 +33196,9 @@ _loop0_120_rule(Parser *p) return _seq; } -// _tmp_121: bitwise_or ((',' bitwise_or))* ','? +// _tmp_121: assignment_expression | expression !':=' static void * _tmp_121_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // bitwise_or ((',' bitwise_or))* ','? - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - asdl_seq * _loop0_257_var; - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings - expr_ty bitwise_or_var; - if ( - (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or - && - (_loop0_257_var = _loop0_257_rule(p)) // ((',' bitwise_or))* - && - (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? - ) - { - D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_257_var, _opt_var); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_122: assignment_expression | expression !':=' -static void * -_tmp_122_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33137,18 +33214,18 @@ _tmp_122_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -33156,7 +33233,7 @@ _tmp_122_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -33164,12 +33241,12 @@ _tmp_122_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -33178,9 +33255,9 @@ _tmp_122_rule(Parser *p) return _res; } -// _loop0_124: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_123: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_124_rule(Parser *p) +_loop0_123_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33205,13 +33282,13 @@ _loop0_124_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_258_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_257_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -33237,7 +33314,7 @@ _loop0_124_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_124[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_123[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33254,10 +33331,10 @@ _loop0_124_rule(Parser *p) return _seq; } -// _gather_123: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124 +// _gather_122: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123 static asdl_seq * -_gather_123_rule(Parser *p) +_gather_122_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33268,27 +33345,27 @@ _gather_123_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124")); + D(fprintf(stderr, "%*c> _gather_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_258_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_257_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_124_rule(p)) // _loop0_124 + (seq = _loop0_123_rule(p)) // _loop0_123 ) { - D(fprintf(stderr, "%*c+ _gather_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124")); + D(fprintf(stderr, "%*c+ _gather_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_123[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_124")); + D(fprintf(stderr, "%*c%s _gather_122[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_123")); } _res = NULL; done: @@ -33296,9 +33373,9 @@ _gather_123_rule(Parser *p) return _res; } -// _tmp_125: ',' kwargs +// _tmp_124: ',' kwargs static void * -_tmp_125_rule(Parser *p) +_tmp_124_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33314,7 +33391,7 @@ _tmp_125_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); Token * _literal; asdl_seq* k; if ( @@ -33323,7 +33400,7 @@ _tmp_125_rule(Parser *p) (k = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); _res = k; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -33333,7 +33410,7 @@ _tmp_125_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwargs")); } _res = NULL; @@ -33342,9 +33419,9 @@ _tmp_125_rule(Parser *p) return _res; } -// _loop0_127: ',' kwarg_or_starred +// _loop0_126: ',' kwarg_or_starred static asdl_seq * -_loop0_127_rule(Parser *p) +_loop0_126_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33369,7 +33446,7 @@ _loop0_127_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); + D(fprintf(stderr, "%*c> _loop0_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -33401,7 +33478,7 @@ _loop0_127_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_127[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_126[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33418,9 +33495,9 @@ _loop0_127_rule(Parser *p) return _seq; } -// _gather_126: kwarg_or_starred _loop0_127 +// _gather_125: kwarg_or_starred _loop0_126 static asdl_seq * -_gather_126_rule(Parser *p) +_gather_125_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33431,27 +33508,27 @@ _gather_126_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_starred _loop0_127 + { // kwarg_or_starred _loop0_126 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_127")); + D(fprintf(stderr, "%*c> _gather_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_126")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred && - (seq = _loop0_127_rule(p)) // _loop0_127 + (seq = _loop0_126_rule(p)) // _loop0_126 ) { - D(fprintf(stderr, "%*c+ _gather_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_127")); + D(fprintf(stderr, "%*c+ _gather_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_126")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_126[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_127")); + D(fprintf(stderr, "%*c%s _gather_125[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_126")); } _res = NULL; done: @@ -33459,9 +33536,9 @@ _gather_126_rule(Parser *p) return _res; } -// _loop0_129: ',' kwarg_or_double_starred +// _loop0_128: ',' kwarg_or_double_starred static asdl_seq * -_loop0_129_rule(Parser *p) +_loop0_128_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33486,7 +33563,7 @@ _loop0_129_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); + D(fprintf(stderr, "%*c> _loop0_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -33518,7 +33595,7 @@ _loop0_129_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_129[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_128[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33535,9 +33612,9 @@ _loop0_129_rule(Parser *p) return _seq; } -// _gather_128: kwarg_or_double_starred _loop0_129 +// _gather_127: kwarg_or_double_starred _loop0_128 static asdl_seq * -_gather_128_rule(Parser *p) +_gather_127_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33548,27 +33625,27 @@ _gather_128_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_double_starred _loop0_129 + { // kwarg_or_double_starred _loop0_128 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_129")); + D(fprintf(stderr, "%*c> _gather_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_128")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred && - (seq = _loop0_129_rule(p)) // _loop0_129 + (seq = _loop0_128_rule(p)) // _loop0_128 ) { - D(fprintf(stderr, "%*c+ _gather_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_129")); + D(fprintf(stderr, "%*c+ _gather_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_128")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_128[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_129")); + D(fprintf(stderr, "%*c%s _gather_127[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_128")); } _res = NULL; done: @@ -33576,9 +33653,9 @@ _gather_128_rule(Parser *p) return _res; } -// _loop0_131: ',' kwarg_or_starred +// _loop0_130: ',' kwarg_or_starred static asdl_seq * -_loop0_131_rule(Parser *p) +_loop0_130_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33603,7 +33680,7 @@ _loop0_131_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); + D(fprintf(stderr, "%*c> _loop0_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -33635,7 +33712,7 @@ _loop0_131_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_131[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_130[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33652,9 +33729,9 @@ _loop0_131_rule(Parser *p) return _seq; } -// _gather_130: kwarg_or_starred _loop0_131 +// _gather_129: kwarg_or_starred _loop0_130 static asdl_seq * -_gather_130_rule(Parser *p) +_gather_129_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33665,27 +33742,27 @@ _gather_130_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_starred _loop0_131 + { // kwarg_or_starred _loop0_130 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_131")); + D(fprintf(stderr, "%*c> _gather_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_130")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred && - (seq = _loop0_131_rule(p)) // _loop0_131 + (seq = _loop0_130_rule(p)) // _loop0_130 ) { - D(fprintf(stderr, "%*c+ _gather_130[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_131")); + D(fprintf(stderr, "%*c+ _gather_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_130")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_130[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_131")); + D(fprintf(stderr, "%*c%s _gather_129[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_130")); } _res = NULL; done: @@ -33693,9 +33770,9 @@ _gather_130_rule(Parser *p) return _res; } -// _loop0_133: ',' kwarg_or_double_starred +// _loop0_132: ',' kwarg_or_double_starred static asdl_seq * -_loop0_133_rule(Parser *p) +_loop0_132_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33720,7 +33797,7 @@ _loop0_133_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); + D(fprintf(stderr, "%*c> _loop0_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -33752,7 +33829,7 @@ _loop0_133_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_133[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_132[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33769,9 +33846,9 @@ _loop0_133_rule(Parser *p) return _seq; } -// _gather_132: kwarg_or_double_starred _loop0_133 +// _gather_131: kwarg_or_double_starred _loop0_132 static asdl_seq * -_gather_132_rule(Parser *p) +_gather_131_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33782,27 +33859,27 @@ _gather_132_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_double_starred _loop0_133 + { // kwarg_or_double_starred _loop0_132 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_133")); + D(fprintf(stderr, "%*c> _gather_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_132")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred && - (seq = _loop0_133_rule(p)) // _loop0_133 + (seq = _loop0_132_rule(p)) // _loop0_132 ) { - D(fprintf(stderr, "%*c+ _gather_132[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_133")); + D(fprintf(stderr, "%*c+ _gather_131[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_132")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_132[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_133")); + D(fprintf(stderr, "%*c%s _gather_131[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_132")); } _res = NULL; done: @@ -33810,9 +33887,9 @@ _gather_132_rule(Parser *p) return _res; } -// _loop0_134: (',' star_target) +// _loop0_133: (',' star_target) static asdl_seq * -_loop0_134_rule(Parser *p) +_loop0_133_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33837,13 +33914,13 @@ _loop0_134_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_259_var; + D(fprintf(stderr, "%*c> _loop0_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_258_var; while ( - (_tmp_259_var = _tmp_259_rule(p)) // ',' star_target + (_tmp_258_var = _tmp_258_rule(p)) // ',' star_target ) { - _res = _tmp_259_var; + _res = _tmp_258_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33860,7 +33937,7 @@ _loop0_134_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_134[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_133[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33877,9 +33954,9 @@ _loop0_134_rule(Parser *p) return _seq; } -// _loop0_136: ',' star_target +// _loop0_135: ',' star_target static asdl_seq * -_loop0_136_rule(Parser *p) +_loop0_135_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33904,7 +33981,7 @@ _loop0_136_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _loop0_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty elem; while ( @@ -33936,7 +34013,7 @@ _loop0_136_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_136[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_135[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33953,9 +34030,9 @@ _loop0_136_rule(Parser *p) return _seq; } -// _gather_135: star_target _loop0_136 +// _gather_134: star_target _loop0_135 static asdl_seq * -_gather_135_rule(Parser *p) +_gather_134_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -33966,27 +34043,27 @@ _gather_135_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // star_target _loop0_136 + { // star_target _loop0_135 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_136")); + D(fprintf(stderr, "%*c> _gather_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_135")); expr_ty elem; asdl_seq * seq; if ( (elem = star_target_rule(p)) // star_target && - (seq = _loop0_136_rule(p)) // _loop0_136 + (seq = _loop0_135_rule(p)) // _loop0_135 ) { - D(fprintf(stderr, "%*c+ _gather_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_136")); + D(fprintf(stderr, "%*c+ _gather_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_135")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_135[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_136")); + D(fprintf(stderr, "%*c%s _gather_134[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_135")); } _res = NULL; done: @@ -33994,9 +34071,9 @@ _gather_135_rule(Parser *p) return _res; } -// _loop1_137: (',' star_target) +// _loop1_136: (',' star_target) static asdl_seq * -_loop1_137_rule(Parser *p) +_loop1_136_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34021,13 +34098,13 @@ _loop1_137_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_260_var; + D(fprintf(stderr, "%*c> _loop1_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_259_var; while ( - (_tmp_260_var = _tmp_260_rule(p)) // ',' star_target + (_tmp_259_var = _tmp_259_rule(p)) // ',' star_target ) { - _res = _tmp_260_var; + _res = _tmp_259_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34044,7 +34121,7 @@ _loop1_137_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_137[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_136[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } if (_n == 0 || p->error_indicator) { @@ -34066,9 +34143,9 @@ _loop1_137_rule(Parser *p) return _seq; } -// _tmp_138: !'*' star_target +// _tmp_137: !'*' star_target static void * -_tmp_138_rule(Parser *p) +_tmp_137_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34084,7 +34161,7 @@ _tmp_138_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + D(fprintf(stderr, "%*c> _tmp_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); expr_ty star_target_var; if ( _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 16) // token='*' @@ -34092,12 +34169,12 @@ _tmp_138_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + D(fprintf(stderr, "%*c+ _tmp_137[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); _res = star_target_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_138[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_137[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!'*' star_target")); } _res = NULL; @@ -34106,9 +34183,9 @@ _tmp_138_rule(Parser *p) return _res; } -// _loop0_140: ',' del_target +// _loop0_139: ',' del_target static asdl_seq * -_loop0_140_rule(Parser *p) +_loop0_139_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34133,7 +34210,7 @@ _loop0_140_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); + D(fprintf(stderr, "%*c> _loop0_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); Token * _literal; expr_ty elem; while ( @@ -34165,7 +34242,7 @@ _loop0_140_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_140[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_139[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' del_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34182,9 +34259,9 @@ _loop0_140_rule(Parser *p) return _seq; } -// _gather_139: del_target _loop0_140 +// _gather_138: del_target _loop0_139 static asdl_seq * -_gather_139_rule(Parser *p) +_gather_138_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34195,27 +34272,27 @@ _gather_139_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // del_target _loop0_140 + { // del_target _loop0_139 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_140")); + D(fprintf(stderr, "%*c> _gather_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_139")); expr_ty elem; asdl_seq * seq; if ( (elem = del_target_rule(p)) // del_target && - (seq = _loop0_140_rule(p)) // _loop0_140 + (seq = _loop0_139_rule(p)) // _loop0_139 ) { - D(fprintf(stderr, "%*c+ _gather_139[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_140")); + D(fprintf(stderr, "%*c+ _gather_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_139")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_139[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_140")); + D(fprintf(stderr, "%*c%s _gather_138[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_139")); } _res = NULL; done: @@ -34223,9 +34300,9 @@ _gather_139_rule(Parser *p) return _res; } -// _loop0_142: ',' expression +// _loop0_141: ',' expression static asdl_seq * -_loop0_142_rule(Parser *p) +_loop0_141_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34250,7 +34327,7 @@ _loop0_142_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -34282,7 +34359,7 @@ _loop0_142_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_142[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_141[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34299,9 +34376,9 @@ _loop0_142_rule(Parser *p) return _seq; } -// _gather_141: expression _loop0_142 +// _gather_140: expression _loop0_141 static asdl_seq * -_gather_141_rule(Parser *p) +_gather_140_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34312,27 +34389,27 @@ _gather_141_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_142 + { // expression _loop0_141 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_142")); + D(fprintf(stderr, "%*c> _gather_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_141")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_142_rule(p)) // _loop0_142 + (seq = _loop0_141_rule(p)) // _loop0_141 ) { - D(fprintf(stderr, "%*c+ _gather_141[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_142")); + D(fprintf(stderr, "%*c+ _gather_140[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_141")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_141[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_142")); + D(fprintf(stderr, "%*c%s _gather_140[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_141")); } _res = NULL; done: @@ -34340,9 +34417,9 @@ _gather_141_rule(Parser *p) return _res; } -// _loop0_144: ',' expression +// _loop0_143: ',' expression static asdl_seq * -_loop0_144_rule(Parser *p) +_loop0_143_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34367,7 +34444,7 @@ _loop0_144_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -34399,7 +34476,7 @@ _loop0_144_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_144[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_143[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34416,9 +34493,9 @@ _loop0_144_rule(Parser *p) return _seq; } -// _gather_143: expression _loop0_144 +// _gather_142: expression _loop0_143 static asdl_seq * -_gather_143_rule(Parser *p) +_gather_142_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34429,27 +34506,27 @@ _gather_143_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_144 + { // expression _loop0_143 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_144")); + D(fprintf(stderr, "%*c> _gather_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_143")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_144_rule(p)) // _loop0_144 + (seq = _loop0_143_rule(p)) // _loop0_143 ) { - D(fprintf(stderr, "%*c+ _gather_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_144")); + D(fprintf(stderr, "%*c+ _gather_142[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_143")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_143[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_144")); + D(fprintf(stderr, "%*c%s _gather_142[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_143")); } _res = NULL; done: @@ -34457,9 +34534,9 @@ _gather_143_rule(Parser *p) return _res; } -// _loop0_146: ',' expression +// _loop0_145: ',' expression static asdl_seq * -_loop0_146_rule(Parser *p) +_loop0_145_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34484,7 +34561,7 @@ _loop0_146_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -34516,7 +34593,7 @@ _loop0_146_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_146[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_145[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34533,9 +34610,9 @@ _loop0_146_rule(Parser *p) return _seq; } -// _gather_145: expression _loop0_146 +// _gather_144: expression _loop0_145 static asdl_seq * -_gather_145_rule(Parser *p) +_gather_144_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34546,27 +34623,27 @@ _gather_145_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_146 + { // expression _loop0_145 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_146")); + D(fprintf(stderr, "%*c> _gather_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_145")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_146_rule(p)) // _loop0_146 + (seq = _loop0_145_rule(p)) // _loop0_145 ) { - D(fprintf(stderr, "%*c+ _gather_145[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_146")); + D(fprintf(stderr, "%*c+ _gather_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_145")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_145[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_146")); + D(fprintf(stderr, "%*c%s _gather_144[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_145")); } _res = NULL; done: @@ -34574,9 +34651,9 @@ _gather_145_rule(Parser *p) return _res; } -// _loop0_148: ',' expression +// _loop0_147: ',' expression static asdl_seq * -_loop0_148_rule(Parser *p) +_loop0_147_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34601,7 +34678,7 @@ _loop0_148_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -34633,7 +34710,7 @@ _loop0_148_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_148[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_147[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34650,9 +34727,9 @@ _loop0_148_rule(Parser *p) return _seq; } -// _gather_147: expression _loop0_148 +// _gather_146: expression _loop0_147 static asdl_seq * -_gather_147_rule(Parser *p) +_gather_146_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34663,27 +34740,27 @@ _gather_147_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_148 + { // expression _loop0_147 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_148")); + D(fprintf(stderr, "%*c> _gather_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_147")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_148_rule(p)) // _loop0_148 + (seq = _loop0_147_rule(p)) // _loop0_147 ) { - D(fprintf(stderr, "%*c+ _gather_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_148")); + D(fprintf(stderr, "%*c+ _gather_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_147")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_147[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_148")); + D(fprintf(stderr, "%*c%s _gather_146[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_147")); } _res = NULL; done: @@ -34691,9 +34768,9 @@ _gather_147_rule(Parser *p) return _res; } -// _tmp_149: NEWLINE INDENT +// _tmp_148: NEWLINE INDENT static void * -_tmp_149_rule(Parser *p) +_tmp_148_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34709,7 +34786,7 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); Token * indent_var; Token * newline_var; if ( @@ -34718,12 +34795,12 @@ _tmp_149_rule(Parser *p) (indent_var = _PyPegen_expect_token(p, INDENT)) // token='INDENT' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); _res = _PyPegen_dummy_name(p, newline_var, indent_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE INDENT")); } _res = NULL; @@ -34732,11 +34809,11 @@ _tmp_149_rule(Parser *p) return _res; } -// _tmp_150: +// _tmp_149: // | (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) // | kwargs static void * -_tmp_150_rule(Parser *p) +_tmp_149_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34752,18 +34829,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - void *_tmp_261_var; + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + void *_tmp_260_var; if ( - (_tmp_261_var = _tmp_261_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs + (_tmp_260_var = _tmp_260_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - _res = _tmp_261_var; + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + _res = _tmp_260_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); } { // kwargs @@ -34771,18 +34848,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); asdl_seq* kwargs_var; if ( (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); _res = kwargs_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwargs")); } _res = NULL; @@ -34791,9 +34868,9 @@ _tmp_150_rule(Parser *p) return _res; } -// _loop0_152: ',' (starred_expression !'=') +// _loop0_151: ',' (starred_expression !'=') static asdl_seq * -_loop0_152_rule(Parser *p) +_loop0_151_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34818,13 +34895,13 @@ _loop0_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression !'=')")); + D(fprintf(stderr, "%*c> _loop0_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_262_rule(p)) // starred_expression !'=' + (elem = _tmp_261_rule(p)) // starred_expression !'=' ) { _res = elem; @@ -34850,7 +34927,7 @@ _loop0_152_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_151[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34867,9 +34944,9 @@ _loop0_152_rule(Parser *p) return _seq; } -// _gather_151: (starred_expression !'=') _loop0_152 +// _gather_150: (starred_expression !'=') _loop0_151 static asdl_seq * -_gather_151_rule(Parser *p) +_gather_150_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34880,27 +34957,27 @@ _gather_151_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression !'=') _loop0_152 + { // (starred_expression !'=') _loop0_151 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_152")); + D(fprintf(stderr, "%*c> _gather_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_151")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_262_rule(p)) // starred_expression !'=' + (elem = _tmp_261_rule(p)) // starred_expression !'=' && - (seq = _loop0_152_rule(p)) // _loop0_152 + (seq = _loop0_151_rule(p)) // _loop0_151 ) { - D(fprintf(stderr, "%*c+ _gather_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_152")); + D(fprintf(stderr, "%*c+ _gather_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_151")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_151[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression !'=') _loop0_152")); + D(fprintf(stderr, "%*c%s _gather_150[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression !'=') _loop0_151")); } _res = NULL; done: @@ -34908,9 +34985,9 @@ _gather_151_rule(Parser *p) return _res; } -// _tmp_153: args | expression for_if_clauses +// _tmp_152: args | expression for_if_clauses static void * -_tmp_153_rule(Parser *p) +_tmp_152_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34926,18 +35003,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); expr_ty args_var; if ( (args_var = args_rule(p)) // args ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); _res = args_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args")); } { // expression for_if_clauses @@ -34945,7 +35022,7 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); expr_ty expression_var; asdl_comprehension_seq* for_if_clauses_var; if ( @@ -34954,12 +35031,12 @@ _tmp_153_rule(Parser *p) (for_if_clauses_var = for_if_clauses_rule(p)) // for_if_clauses ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); _res = _PyPegen_dummy_name(p, expression_var, for_if_clauses_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression for_if_clauses")); } _res = NULL; @@ -34968,9 +35045,9 @@ _tmp_153_rule(Parser *p) return _res; } -// _tmp_154: args ',' +// _tmp_153: args ',' static void * -_tmp_154_rule(Parser *p) +_tmp_153_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -34986,7 +35063,7 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); Token * _literal; expr_ty args_var; if ( @@ -34995,12 +35072,12 @@ _tmp_154_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); _res = _PyPegen_dummy_name(p, args_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args ','")); } _res = NULL; @@ -35009,9 +35086,9 @@ _tmp_154_rule(Parser *p) return _res; } -// _tmp_155: ',' | ')' +// _tmp_154: ',' | ')' static void * -_tmp_155_rule(Parser *p) +_tmp_154_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35027,18 +35104,18 @@ _tmp_155_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -35046,18 +35123,18 @@ _tmp_155_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } _res = NULL; @@ -35066,9 +35143,9 @@ _tmp_155_rule(Parser *p) return _res; } -// _tmp_156: 'True' | 'False' | 'None' +// _tmp_155: 'True' | 'False' | 'None' static void * -_tmp_156_rule(Parser *p) +_tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35084,18 +35161,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 613)) // token='True' + (_keyword = _PyPegen_expect_token(p, 610)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'False' @@ -35103,18 +35180,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 615)) // token='False' + (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } { // 'None' @@ -35122,18 +35199,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 614)) // token='None' + (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } _res = NULL; @@ -35142,9 +35219,9 @@ _tmp_156_rule(Parser *p) return _res; } -// _tmp_157: NAME '=' +// _tmp_156: NAME '=' static void * -_tmp_157_rule(Parser *p) +_tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35160,7 +35237,7 @@ _tmp_157_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); Token * _literal; expr_ty name_var; if ( @@ -35169,12 +35246,12 @@ _tmp_157_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); _res = _PyPegen_dummy_name(p, name_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME '='")); } _res = NULL; @@ -35183,9 +35260,9 @@ _tmp_157_rule(Parser *p) return _res; } -// _tmp_158: NAME STRING | SOFT_KEYWORD +// _tmp_157: NAME STRING | SOFT_KEYWORD static void * -_tmp_158_rule(Parser *p) +_tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35201,7 +35278,7 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); expr_ty name_var; expr_ty string_var; if ( @@ -35210,12 +35287,12 @@ _tmp_158_rule(Parser *p) (string_var = _PyPegen_string_token(p)) // STRING ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); _res = _PyPegen_dummy_name(p, name_var, string_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME STRING")); } { // SOFT_KEYWORD @@ -35223,18 +35300,18 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); expr_ty soft_keyword_var; if ( (soft_keyword_var = _PyPegen_soft_keyword_token(p)) // SOFT_KEYWORD ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); _res = soft_keyword_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "SOFT_KEYWORD")); } _res = NULL; @@ -35243,9 +35320,9 @@ _tmp_158_rule(Parser *p) return _res; } -// _tmp_159: 'else' | ':' +// _tmp_158: 'else' | ':' static void * -_tmp_159_rule(Parser *p) +_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35261,18 +35338,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 665)) // token='else' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'else'")); } { // ':' @@ -35280,18 +35357,18 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -35300,9 +35377,9 @@ _tmp_159_rule(Parser *p) return _res; } -// _tmp_160: '=' | ':=' +// _tmp_159: '=' | ':=' static void * -_tmp_160_rule(Parser *p) +_tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35318,18 +35395,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -35337,18 +35414,18 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -35357,9 +35434,9 @@ _tmp_160_rule(Parser *p) return _res; } -// _tmp_161: list | tuple | genexp | 'True' | 'None' | 'False' +// _tmp_160: list | tuple | genexp | 'True' | 'None' | 'False' static void * -_tmp_161_rule(Parser *p) +_tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35375,18 +35452,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); expr_ty list_var; if ( (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } { // tuple @@ -35394,18 +35471,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); expr_ty tuple_var; if ( (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } { // genexp @@ -35413,18 +35490,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); expr_ty genexp_var; if ( (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } { // 'True' @@ -35432,18 +35509,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 613)) // token='True' + (_keyword = _PyPegen_expect_token(p, 610)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'None' @@ -35451,18 +35528,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 614)) // token='None' + (_keyword = _PyPegen_expect_token(p, 611)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } { // 'False' @@ -35470,18 +35547,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 615)) // token='False' + (_keyword = _PyPegen_expect_token(p, 612)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } _res = NULL; @@ -35490,9 +35567,9 @@ _tmp_161_rule(Parser *p) return _res; } -// _tmp_162: '=' | ':=' +// _tmp_161: '=' | ':=' static void * -_tmp_162_rule(Parser *p) +_tmp_161_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35508,18 +35585,18 @@ _tmp_162_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -35527,18 +35604,18 @@ _tmp_162_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -35547,9 +35624,9 @@ _tmp_162_rule(Parser *p) return _res; } -// _loop0_163: star_named_expressions +// _loop0_162: star_named_expressions static asdl_seq * -_loop0_163_rule(Parser *p) +_loop0_162_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35574,7 +35651,7 @@ _loop0_163_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); + D(fprintf(stderr, "%*c> _loop0_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); asdl_expr_seq* star_named_expressions_var; while ( (star_named_expressions_var = star_named_expressions_rule(p)) // star_named_expressions @@ -35597,7 +35674,7 @@ _loop0_163_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_163[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expressions")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35614,9 +35691,9 @@ _loop0_163_rule(Parser *p) return _seq; } -// _loop0_164: (star_targets '=') +// _loop0_163: (star_targets '=') static asdl_seq * -_loop0_164_rule(Parser *p) +_loop0_163_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35641,13 +35718,13 @@ _loop0_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_263_var; + D(fprintf(stderr, "%*c> _loop0_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_262_var; while ( - (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' + (_tmp_262_var = _tmp_262_rule(p)) // star_targets '=' ) { - _res = _tmp_263_var; + _res = _tmp_262_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35664,7 +35741,7 @@ _loop0_164_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35681,9 +35758,9 @@ _loop0_164_rule(Parser *p) return _seq; } -// _loop0_165: (star_targets '=') +// _loop0_164: (star_targets '=') static asdl_seq * -_loop0_165_rule(Parser *p) +_loop0_164_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35708,13 +35785,13 @@ _loop0_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_264_var; + D(fprintf(stderr, "%*c> _loop0_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_263_var; while ( - (_tmp_264_var = _tmp_264_rule(p)) // star_targets '=' + (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' ) { - _res = _tmp_264_var; + _res = _tmp_263_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35731,7 +35808,7 @@ _loop0_165_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35748,9 +35825,9 @@ _loop0_165_rule(Parser *p) return _seq; } -// _tmp_166: '[' | '(' | '{' +// _tmp_165: '[' | '(' | '{' static void * -_tmp_166_rule(Parser *p) +_tmp_165_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35766,18 +35843,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '(' @@ -35785,18 +35862,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '{' @@ -35804,18 +35881,18 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35824,9 +35901,9 @@ _tmp_166_rule(Parser *p) return _res; } -// _tmp_167: '[' | '{' +// _tmp_166: '[' | '{' static void * -_tmp_167_rule(Parser *p) +_tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35842,18 +35919,18 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -35861,18 +35938,18 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35881,9 +35958,9 @@ _tmp_167_rule(Parser *p) return _res; } -// _tmp_168: '[' | '{' +// _tmp_167: '[' | '{' static void * -_tmp_168_rule(Parser *p) +_tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35899,18 +35976,18 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -35918,18 +35995,18 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -35938,9 +36015,9 @@ _tmp_168_rule(Parser *p) return _res; } -// _tmp_169: slash_no_default | slash_with_default +// _tmp_168: slash_no_default | slash_with_default static void * -_tmp_169_rule(Parser *p) +_tmp_168_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -35956,18 +36033,18 @@ _tmp_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -35975,18 +36052,18 @@ _tmp_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -35995,9 +36072,9 @@ _tmp_169_rule(Parser *p) return _res; } -// _loop0_170: param_maybe_default +// _loop0_169: param_maybe_default static asdl_seq * -_loop0_170_rule(Parser *p) +_loop0_169_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36022,7 +36099,7 @@ _loop0_170_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36045,7 +36122,7 @@ _loop0_170_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_170[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36062,9 +36139,9 @@ _loop0_170_rule(Parser *p) return _seq; } -// _loop0_171: param_no_default +// _loop0_170: param_no_default static asdl_seq * -_loop0_171_rule(Parser *p) +_loop0_170_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36089,7 +36166,7 @@ _loop0_171_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -36112,7 +36189,7 @@ _loop0_171_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_170[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36129,9 +36206,9 @@ _loop0_171_rule(Parser *p) return _seq; } -// _loop0_172: param_no_default +// _loop0_171: param_no_default static asdl_seq * -_loop0_172_rule(Parser *p) +_loop0_171_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36156,7 +36233,7 @@ _loop0_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -36179,7 +36256,7 @@ _loop0_172_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36196,9 +36273,9 @@ _loop0_172_rule(Parser *p) return _seq; } -// _loop1_173: param_no_default +// _loop1_172: param_no_default static asdl_seq * -_loop1_173_rule(Parser *p) +_loop1_172_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36223,7 +36300,7 @@ _loop1_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop1_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -36246,7 +36323,7 @@ _loop1_173_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_172[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } if (_n == 0 || p->error_indicator) { @@ -36268,9 +36345,9 @@ _loop1_173_rule(Parser *p) return _seq; } -// _tmp_174: slash_no_default | slash_with_default +// _tmp_173: slash_no_default | slash_with_default static void * -_tmp_174_rule(Parser *p) +_tmp_173_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36286,18 +36363,18 @@ _tmp_174_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -36305,18 +36382,18 @@ _tmp_174_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -36325,9 +36402,9 @@ _tmp_174_rule(Parser *p) return _res; } -// _loop0_175: param_maybe_default +// _loop0_174: param_maybe_default static asdl_seq * -_loop0_175_rule(Parser *p) +_loop0_174_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36352,7 +36429,7 @@ _loop0_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36375,7 +36452,7 @@ _loop0_175_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36392,9 +36469,9 @@ _loop0_175_rule(Parser *p) return _seq; } -// _tmp_176: ',' | param_no_default +// _tmp_175: ',' | param_no_default static void * -_tmp_176_rule(Parser *p) +_tmp_175_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36410,18 +36487,18 @@ _tmp_176_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // param_no_default @@ -36429,18 +36506,18 @@ _tmp_176_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } _res = NULL; @@ -36449,9 +36526,9 @@ _tmp_176_rule(Parser *p) return _res; } -// _loop0_177: param_maybe_default +// _loop0_176: param_maybe_default static asdl_seq * -_loop0_177_rule(Parser *p) +_loop0_176_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36476,7 +36553,7 @@ _loop0_177_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36499,7 +36576,7 @@ _loop0_177_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_177[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_176[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36516,9 +36593,9 @@ _loop0_177_rule(Parser *p) return _seq; } -// _loop1_178: param_maybe_default +// _loop1_177: param_maybe_default static asdl_seq * -_loop1_178_rule(Parser *p) +_loop1_177_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36543,7 +36620,7 @@ _loop1_178_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36566,7 +36643,7 @@ _loop1_178_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_178[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_177[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -36588,9 +36665,9 @@ _loop1_178_rule(Parser *p) return _seq; } -// _tmp_179: ')' | ',' +// _tmp_178: ')' | ',' static void * -_tmp_179_rule(Parser *p) +_tmp_178_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36606,18 +36683,18 @@ _tmp_179_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' @@ -36625,18 +36702,18 @@ _tmp_179_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_178[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_178[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_178[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36645,9 +36722,9 @@ _tmp_179_rule(Parser *p) return _res; } -// _tmp_180: ')' | ',' (')' | '**') +// _tmp_179: ')' | ',' (')' | '**') static void * -_tmp_180_rule(Parser *p) +_tmp_179_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36663,18 +36740,18 @@ _tmp_180_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' (')' | '**') @@ -36682,21 +36759,21 @@ _tmp_180_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + D(fprintf(stderr, "%*c> _tmp_179[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_265_var; + void *_tmp_264_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_265_var = _tmp_265_rule(p)) // ')' | '**' + (_tmp_264_var = _tmp_264_rule(p)) // ')' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); + D(fprintf(stderr, "%*c+ _tmp_179[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_264_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_179[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (')' | '**')")); } _res = NULL; @@ -36705,9 +36782,9 @@ _tmp_180_rule(Parser *p) return _res; } -// _tmp_181: param_no_default | ',' +// _tmp_180: param_no_default | ',' static void * -_tmp_181_rule(Parser *p) +_tmp_180_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36723,18 +36800,18 @@ _tmp_181_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -36742,18 +36819,18 @@ _tmp_181_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_181[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_181[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_180[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36762,9 +36839,9 @@ _tmp_181_rule(Parser *p) return _res; } -// _loop0_182: param_maybe_default +// _loop0_181: param_maybe_default static asdl_seq * -_loop0_182_rule(Parser *p) +_loop0_181_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36789,7 +36866,7 @@ _loop0_182_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_181[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -36812,7 +36889,7 @@ _loop0_182_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_182[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_181[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36829,9 +36906,9 @@ _loop0_182_rule(Parser *p) return _seq; } -// _tmp_183: param_no_default | ',' +// _tmp_182: param_no_default | ',' static void * -_tmp_183_rule(Parser *p) +_tmp_182_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36847,18 +36924,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -36866,18 +36943,18 @@ _tmp_183_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_182[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_182[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_182[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36886,9 +36963,9 @@ _tmp_183_rule(Parser *p) return _res; } -// _tmp_184: '*' | '**' | '/' +// _tmp_183: '*' | '**' | '/' static void * -_tmp_184_rule(Parser *p) +_tmp_183_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36904,18 +36981,18 @@ _tmp_184_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -36923,18 +37000,18 @@ _tmp_184_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -36942,18 +37019,18 @@ _tmp_184_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_183[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_184[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_183[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_184[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_183[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -36962,9 +37039,9 @@ _tmp_184_rule(Parser *p) return _res; } -// _loop1_185: param_with_default +// _loop1_184: param_with_default static asdl_seq * -_loop1_185_rule(Parser *p) +_loop1_184_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -36989,7 +37066,7 @@ _loop1_185_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop1_184[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -37012,7 +37089,7 @@ _loop1_185_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_185[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_184[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -37034,9 +37111,9 @@ _loop1_185_rule(Parser *p) return _seq; } -// _tmp_186: lambda_slash_no_default | lambda_slash_with_default +// _tmp_185: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_186_rule(Parser *p) +_tmp_185_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37052,18 +37129,18 @@ _tmp_186_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_186[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_186[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -37071,18 +37148,18 @@ _tmp_186_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_185[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_186[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_185[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_186[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_185[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -37091,9 +37168,9 @@ _tmp_186_rule(Parser *p) return _res; } -// _loop0_187: lambda_param_maybe_default +// _loop0_186: lambda_param_maybe_default static asdl_seq * -_loop0_187_rule(Parser *p) +_loop0_186_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37118,7 +37195,7 @@ _loop0_187_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_186[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37141,7 +37218,7 @@ _loop0_187_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_187[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_186[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37158,9 +37235,9 @@ _loop0_187_rule(Parser *p) return _seq; } -// _loop0_188: lambda_param_no_default +// _loop0_187: lambda_param_no_default static asdl_seq * -_loop0_188_rule(Parser *p) +_loop0_187_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37185,7 +37262,7 @@ _loop0_188_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_187[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -37208,7 +37285,7 @@ _loop0_188_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_188[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_187[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37225,9 +37302,9 @@ _loop0_188_rule(Parser *p) return _seq; } -// _loop0_189: lambda_param_no_default +// _loop0_188: lambda_param_no_default static asdl_seq * -_loop0_189_rule(Parser *p) +_loop0_188_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37252,7 +37329,7 @@ _loop0_189_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_188[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -37275,7 +37352,7 @@ _loop0_189_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_189[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_188[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37292,9 +37369,9 @@ _loop0_189_rule(Parser *p) return _seq; } -// _loop0_191: ',' lambda_param +// _loop0_190: ',' lambda_param static asdl_seq * -_loop0_191_rule(Parser *p) +_loop0_190_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37319,7 +37396,7 @@ _loop0_191_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); + D(fprintf(stderr, "%*c> _loop0_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); Token * _literal; arg_ty elem; while ( @@ -37351,7 +37428,7 @@ _loop0_191_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_191[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_190[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' lambda_param")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37368,9 +37445,9 @@ _loop0_191_rule(Parser *p) return _seq; } -// _gather_190: lambda_param _loop0_191 +// _gather_189: lambda_param _loop0_190 static asdl_seq * -_gather_190_rule(Parser *p) +_gather_189_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37381,27 +37458,27 @@ _gather_190_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // lambda_param _loop0_191 + { // lambda_param _loop0_190 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_190[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_191")); + D(fprintf(stderr, "%*c> _gather_189[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_190")); arg_ty elem; asdl_seq * seq; if ( (elem = lambda_param_rule(p)) // lambda_param && - (seq = _loop0_191_rule(p)) // _loop0_191 + (seq = _loop0_190_rule(p)) // _loop0_190 ) { - D(fprintf(stderr, "%*c+ _gather_190[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_191")); + D(fprintf(stderr, "%*c+ _gather_189[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_190")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_190[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_191")); + D(fprintf(stderr, "%*c%s _gather_189[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_190")); } _res = NULL; done: @@ -37409,9 +37486,9 @@ _gather_190_rule(Parser *p) return _res; } -// _tmp_192: lambda_slash_no_default | lambda_slash_with_default +// _tmp_191: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_192_rule(Parser *p) +_tmp_191_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37427,18 +37504,18 @@ _tmp_192_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_191[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_192[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_191[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -37446,18 +37523,18 @@ _tmp_192_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_191[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_192[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_191[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_192[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_191[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -37466,9 +37543,9 @@ _tmp_192_rule(Parser *p) return _res; } -// _loop0_193: lambda_param_maybe_default +// _loop0_192: lambda_param_maybe_default static asdl_seq * -_loop0_193_rule(Parser *p) +_loop0_192_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37493,7 +37570,7 @@ _loop0_193_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_192[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37516,7 +37593,7 @@ _loop0_193_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_193[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_192[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37533,9 +37610,9 @@ _loop0_193_rule(Parser *p) return _seq; } -// _tmp_194: ',' | lambda_param_no_default +// _tmp_193: ',' | lambda_param_no_default static void * -_tmp_194_rule(Parser *p) +_tmp_193_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37551,18 +37628,18 @@ _tmp_194_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_194[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_193[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_194[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_193[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // lambda_param_no_default @@ -37570,18 +37647,18 @@ _tmp_194_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_193[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_194[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_193[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_194[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_193[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } _res = NULL; @@ -37590,9 +37667,9 @@ _tmp_194_rule(Parser *p) return _res; } -// _loop0_195: lambda_param_maybe_default +// _loop0_194: lambda_param_maybe_default static asdl_seq * -_loop0_195_rule(Parser *p) +_loop0_194_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37617,7 +37694,7 @@ _loop0_195_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_194[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37640,7 +37717,7 @@ _loop0_195_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_195[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_194[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37657,9 +37734,9 @@ _loop0_195_rule(Parser *p) return _seq; } -// _loop1_196: lambda_param_maybe_default +// _loop1_195: lambda_param_maybe_default static asdl_seq * -_loop1_196_rule(Parser *p) +_loop1_195_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37684,7 +37761,7 @@ _loop1_196_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_195[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37707,7 +37784,7 @@ _loop1_196_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_196[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_195[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -37729,9 +37806,9 @@ _loop1_196_rule(Parser *p) return _seq; } -// _loop1_197: lambda_param_with_default +// _loop1_196: lambda_param_with_default static asdl_seq * -_loop1_197_rule(Parser *p) +_loop1_196_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37756,7 +37833,7 @@ _loop1_197_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c> _loop1_196[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); NameDefaultPair* lambda_param_with_default_var; while ( (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default @@ -37779,7 +37856,7 @@ _loop1_197_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_197[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_196[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -37801,9 +37878,9 @@ _loop1_197_rule(Parser *p) return _seq; } -// _tmp_198: ':' | ',' (':' | '**') +// _tmp_197: ':' | ',' (':' | '**') static void * -_tmp_198_rule(Parser *p) +_tmp_197_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37819,18 +37896,18 @@ _tmp_198_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // ',' (':' | '**') @@ -37838,21 +37915,21 @@ _tmp_198_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + D(fprintf(stderr, "%*c> _tmp_197[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_266_var; + void *_tmp_265_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_266_var = _tmp_266_rule(p)) // ':' | '**' + (_tmp_265_var = _tmp_265_rule(p)) // ':' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_266_var); + D(fprintf(stderr, "%*c+ _tmp_197[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_197[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (':' | '**')")); } _res = NULL; @@ -37861,9 +37938,9 @@ _tmp_198_rule(Parser *p) return _res; } -// _tmp_199: lambda_param_no_default | ',' +// _tmp_198: lambda_param_no_default | ',' static void * -_tmp_199_rule(Parser *p) +_tmp_198_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37879,18 +37956,18 @@ _tmp_199_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_199[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -37898,18 +37975,18 @@ _tmp_199_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_199[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_199[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_198[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -37918,9 +37995,9 @@ _tmp_199_rule(Parser *p) return _res; } -// _loop0_200: lambda_param_maybe_default +// _loop0_199: lambda_param_maybe_default static asdl_seq * -_loop0_200_rule(Parser *p) +_loop0_199_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -37945,7 +38022,7 @@ _loop0_200_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_199[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -37968,7 +38045,7 @@ _loop0_200_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_200[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_199[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37985,9 +38062,9 @@ _loop0_200_rule(Parser *p) return _seq; } -// _tmp_201: lambda_param_no_default | ',' +// _tmp_200: lambda_param_no_default | ',' static void * -_tmp_201_rule(Parser *p) +_tmp_200_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38003,18 +38080,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -38022,18 +38099,18 @@ _tmp_201_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_200[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_200[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_200[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -38042,9 +38119,9 @@ _tmp_201_rule(Parser *p) return _res; } -// _tmp_202: '*' | '**' | '/' +// _tmp_201: '*' | '**' | '/' static void * -_tmp_202_rule(Parser *p) +_tmp_201_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38060,18 +38137,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -38079,18 +38156,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -38098,18 +38175,18 @@ _tmp_202_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_201[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_201[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_201[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -38118,9 +38195,9 @@ _tmp_202_rule(Parser *p) return _res; } -// _tmp_203: ',' | ')' | ':' +// _tmp_202: ',' | ')' | ':' static void * -_tmp_203_rule(Parser *p) +_tmp_202_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -38136,18 +38213,18 @@ _tmp_203_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -38155,18 +38232,18 @@ _tmp_203_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ':' @@ -38174,18 +38251,18 @@ _tmp_203_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_202[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_202[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_202[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -38194,6 +38271,51 @@ _tmp_203_rule(Parser *p) return _res; } +// _tmp_203: bitwise_or ((',' bitwise_or))* ','? +static void * +_tmp_203_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // bitwise_or ((',' bitwise_or))* ','? + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_203[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + asdl_seq * _loop0_266_var; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + expr_ty bitwise_or_var; + if ( + (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or + && + (_loop0_266_var = _loop0_266_rule(p)) // ((',' bitwise_or))* + && + (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? + ) + { + D(fprintf(stderr, "%*c+ _tmp_203[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_266_var, _opt_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_203[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // _loop0_205: ',' dotted_name static asdl_seq * _loop0_205_rule(Parser *p) @@ -41164,76 +41286,9 @@ _tmp_256_rule(Parser *p) return _res; } -// _loop0_257: (',' bitwise_or) -static asdl_seq * -_loop0_257_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // (',' bitwise_or) - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); - void *_tmp_272_var; - while ( - (_tmp_272_var = _tmp_272_rule(p)) // ',' bitwise_or - ) - { - _res = _tmp_272_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_257[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _tmp_258: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_257: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_258_rule(Parser *p) +_tmp_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41249,18 +41304,18 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -41268,20 +41323,20 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_273_var; + D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_272_var; if ( - (_tmp_273_var = _tmp_273_rule(p)) // assignment_expression | expression !':=' + (_tmp_272_var = _tmp_272_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_273_var; + D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_272_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -41290,9 +41345,9 @@ _tmp_258_rule(Parser *p) return _res; } -// _tmp_259: ',' star_target +// _tmp_258: ',' star_target static void * -_tmp_259_rule(Parser *p) +_tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41308,7 +41363,7 @@ _tmp_259_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -41317,7 +41372,7 @@ _tmp_259_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41327,7 +41382,7 @@ _tmp_259_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -41336,9 +41391,9 @@ _tmp_259_rule(Parser *p) return _res; } -// _tmp_260: ',' star_target +// _tmp_259: ',' star_target static void * -_tmp_260_rule(Parser *p) +_tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41354,7 +41409,7 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -41363,7 +41418,7 @@ _tmp_260_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41373,7 +41428,7 @@ _tmp_260_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -41382,10 +41437,10 @@ _tmp_260_rule(Parser *p) return _res; } -// _tmp_261: +// _tmp_260: // | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs static void * -_tmp_261_rule(Parser *p) +_tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41401,24 +41456,24 @@ _tmp_261_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - asdl_seq * _gather_274_var; + D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + asdl_seq * _gather_273_var; Token * _literal; asdl_seq* kwargs_var; if ( - (_gather_274_var = _gather_274_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (_gather_273_var = _gather_273_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - _res = _PyPegen_dummy_name(p, _gather_274_var, _literal, kwargs_var); + D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + _res = _PyPegen_dummy_name(p, _gather_273_var, _literal, kwargs_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); } _res = NULL; @@ -41427,9 +41482,9 @@ _tmp_261_rule(Parser *p) return _res; } -// _tmp_262: starred_expression !'=' +// _tmp_261: starred_expression !'=' static void * -_tmp_262_rule(Parser *p) +_tmp_261_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41445,7 +41500,7 @@ _tmp_262_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression @@ -41453,12 +41508,12 @@ _tmp_262_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression !'='")); } _res = NULL; @@ -41467,9 +41522,9 @@ _tmp_262_rule(Parser *p) return _res; } -// _tmp_263: star_targets '=' +// _tmp_262: star_targets '=' static void * -_tmp_263_rule(Parser *p) +_tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41485,7 +41540,7 @@ _tmp_263_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41494,12 +41549,12 @@ _tmp_263_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41508,9 +41563,9 @@ _tmp_263_rule(Parser *p) return _res; } -// _tmp_264: star_targets '=' +// _tmp_263: star_targets '=' static void * -_tmp_264_rule(Parser *p) +_tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41526,7 +41581,7 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41535,12 +41590,12 @@ _tmp_264_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41549,9 +41604,9 @@ _tmp_264_rule(Parser *p) return _res; } -// _tmp_265: ')' | '**' +// _tmp_264: ')' | '**' static void * -_tmp_265_rule(Parser *p) +_tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41567,18 +41622,18 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -41586,18 +41641,18 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41606,9 +41661,9 @@ _tmp_265_rule(Parser *p) return _res; } -// _tmp_266: ':' | '**' +// _tmp_265: ':' | '**' static void * -_tmp_266_rule(Parser *p) +_tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41624,18 +41679,18 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -41643,18 +41698,18 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41663,6 +41718,73 @@ _tmp_266_rule(Parser *p) return _res; } +// _loop0_266: (',' bitwise_or) +static asdl_seq * +_loop0_266_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // (',' bitwise_or) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop0_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); + void *_tmp_275_var; + while ( + (_tmp_275_var = _tmp_275_rule(p)) // ',' bitwise_or + ) + { + _res = _tmp_275_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_266[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + // _tmp_267: expression ['as' star_target] static void * _tmp_267_rule(Parser *p) @@ -41872,50 +41994,9 @@ _tmp_271_rule(Parser *p) return _res; } -// _tmp_272: ',' bitwise_or +// _tmp_272: assignment_expression | expression !':=' static void * _tmp_272_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ',' bitwise_or - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); - Token * _literal; - expr_ty bitwise_or_var; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or - ) - { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); - _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_273: assignment_expression | expression !':=' -static void * -_tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41931,18 +42012,18 @@ _tmp_273_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -41950,7 +42031,7 @@ _tmp_273_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -41958,12 +42039,12 @@ _tmp_273_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -41972,9 +42053,9 @@ _tmp_273_rule(Parser *p) return _res; } -// _loop0_275: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_274: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_275_rule(Parser *p) +_loop0_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41999,7 +42080,7 @@ _loop0_275_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( @@ -42031,7 +42112,7 @@ _loop0_275_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_275[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_274[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -42048,10 +42129,10 @@ _loop0_275_rule(Parser *p) return _seq; } -// _gather_274: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275 +// _gather_273: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274 static asdl_seq * -_gather_274_rule(Parser *p) +_gather_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42062,27 +42143,68 @@ _gather_274_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); + D(fprintf(stderr, "%*c> _gather_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); void *elem; asdl_seq * seq; if ( (elem = _tmp_280_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_275_rule(p)) // _loop0_275 + (seq = _loop0_274_rule(p)) // _loop0_274 ) { - D(fprintf(stderr, "%*c+ _gather_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); + D(fprintf(stderr, "%*c+ _gather_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_274[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); + D(fprintf(stderr, "%*c%s _gather_273[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_275: ',' bitwise_or +static void * +_tmp_275_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // ',' bitwise_or + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + Token * _literal; + expr_ty bitwise_or_var; + if ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or + ) + { + D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); } _res = NULL; done: diff --git a/Tools/peg_generator/pegen/validator.py b/Tools/peg_generator/pegen/validator.py index c48a01eedf5d5c..4699d5712d9522 100644 --- a/Tools/peg_generator/pegen/validator.py +++ b/Tools/peg_generator/pegen/validator.py @@ -34,6 +34,18 @@ def check_intersection(self, first_alt: Alt, second_alt: Alt) -> None: ) +class RaiseRuleValidator(GrammarValidator): + def visit_Alt(self, node: Alt) -> None: + if self.rulename and self.rulename.startswith('invalid'): + # raising is allowed in invalid rules + return + if node.action and 'RAISE_SYNTAX_ERROR' in node.action: + raise ValidationError( + f"In {self.rulename!r} there is an alternative that contains " + f"RAISE_SYNTAX_ERROR; this is only allowed in invalid_ rules" + ) + + def validate_grammar(the_grammar: grammar.Grammar) -> None: for validator_cls in GrammarValidator.__subclasses__(): validator = validator_cls(the_grammar) From db009348b4b7a4b0aec39472ea074c1b5feeba9b Mon Sep 17 00:00:00 2001 From: Lincoln <71312724+Lincoln-developer@users.noreply.github.com> Date: Thu, 30 May 2024 11:51:23 +0300 Subject: [PATCH 276/903] gh-118055: Update the finder glossary entry (GH-118278) --- Doc/glossary.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 42e2a6f4e301b6..1e5bafce861e52 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -425,11 +425,11 @@ Glossary An object that tries to find the :term:`loader` for a module that is being imported. - Since Python 3.3, there are two types of finder: :term:`meta path finders + There are two types of finder: :term:`meta path finders ` for use with :data:`sys.meta_path`, and :term:`path entry finders ` for use with :data:`sys.path_hooks`. - See :pep:`302`, :pep:`420` and :pep:`451` for much more detail. + See :ref:`importsystem` and :mod:`importlib` for much more detail. floor division Mathematical division that rounds down to nearest integer. The floor From b1374aa1c2e68becf9b6dcdcb8a586b0bd997c0d Mon Sep 17 00:00:00 2001 From: Bradley Reynolds Date: Thu, 30 May 2024 07:21:37 -0500 Subject: [PATCH 277/903] gh-110383: Remove references to removed popen[234] (GH-112783) Signed-off-by: Bradley Reynolds --- Doc/faq/library.rst | 78 -------------------------------------- Doc/library/subprocess.rst | 70 +--------------------------------- 2 files changed, 2 insertions(+), 146 deletions(-) diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index b959cd73921428..a2900952d7bef6 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -541,84 +541,6 @@ Thus, to read *n* bytes from a pipe *p* created with :func:`os.popen`, you need use ``p.read(n)``. -.. XXX update to use subprocess. See the :ref:`subprocess-replacements` section. - - How do I run a subprocess with pipes connected to both input and output? - ------------------------------------------------------------------------ - - Use the :mod:`popen2` module. For example:: - - import popen2 - fromchild, tochild = popen2.popen2("command") - tochild.write("input\n") - tochild.flush() - output = fromchild.readline() - - Warning: in general it is unwise to do this because you can easily cause a - deadlock where your process is blocked waiting for output from the child - while the child is blocked waiting for input from you. This can be caused - by the parent expecting the child to output more text than it does or - by data being stuck in stdio buffers due to lack of flushing. - The Python parent can of course explicitly flush the data it sends to the - child before it reads any output, but if the child is a naive C program it - may have been written to never explicitly flush its output, even if it is - interactive, since flushing is normally automatic. - - Note that a deadlock is also possible if you use :func:`popen3` to read - stdout and stderr. If one of the two is too large for the internal buffer - (increasing the buffer size does not help) and you ``read()`` the other one - first, there is a deadlock, too. - - Note on a bug in popen2: unless your program calls ``wait()`` or - ``waitpid()``, finished child processes are never removed, and eventually - calls to popen2 will fail because of a limit on the number of child - processes. Calling :func:`os.waitpid` with the :const:`os.WNOHANG` option can - prevent this; a good place to insert such a call would be before calling - ``popen2`` again. - - In many cases, all you really need is to run some data through a command and - get the result back. Unless the amount of data is very large, the easiest - way to do this is to write it to a temporary file and run the command with - that temporary file as input. The standard module :mod:`tempfile` exports a - :func:`~tempfile.mktemp` function to generate unique temporary file names. :: - - import tempfile - import os - - class Popen3: - """ - This is a deadlock-safe version of popen that returns - an object with errorlevel, out (a string) and err (a string). - (capturestderr may not work under windows.) - Example: print(Popen3('grep spam','\n\nhere spam\n\n').out) - """ - def __init__(self,command,input=None,capturestderr=None): - outfile=tempfile.mktemp() - command="( %s ) > %s" % (command,outfile) - if input: - infile=tempfile.mktemp() - open(infile,"w").write(input) - command=command+" <"+infile - if capturestderr: - errfile=tempfile.mktemp() - command=command+" 2>"+errfile - self.errorlevel=os.system(command) >> 8 - self.out=open(outfile,"r").read() - os.remove(outfile) - if input: - os.remove(infile) - if capturestderr: - self.err=open(errfile,"r").read() - os.remove(errfile) - - Note that many interactive programs (e.g. vi) don't work well with pipes - substituted for standard input and output. You will have to use pseudo ttys - ("ptys") instead of pipes. Or you can use a Python interface to Don Libes' - "expect" library. A Python extension that interfaces to expect is called - "expy" and available from https://expectpy.sourceforge.net. A pure Python - solution that works like expect is :pypi:`pexpect`. - - How do I access the serial (RS232) port? ---------------------------------------- diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 3a2178e2b7b839..f520d989e0c70d 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1443,36 +1443,8 @@ Environment example:: -Replacing :func:`os.popen`, :func:`os.popen2`, :func:`os.popen3` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdin, child_stdout) = (p.stdin, p.stdout) - -:: - - (child_stdin, - child_stdout, - child_stderr) = os.popen3(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) - (child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - -:: - - (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) - (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) +Replacing :func:`os.popen` +^^^^^^^^^^^^^^^^^^^^^^^^^^ Return code handling translates as follows:: @@ -1489,44 +1461,6 @@ Return code handling translates as follows:: print("There were some errors") -Replacing functions from the :mod:`!popen2` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. note:: - - If the cmd argument to popen2 functions is a string, the command is executed - through /bin/sh. If it is a list, the command is directly executed. - -:: - - (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) - ==> - p = Popen("somestring", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:: - - (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode) - ==> - p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:class:`popen2.Popen3` and :class:`popen2.Popen4` basically work as -:class:`subprocess.Popen`, except that: - -* :class:`Popen` raises an exception if the execution fails. - -* The *capturestderr* argument is replaced with the *stderr* argument. - -* ``stdin=PIPE`` and ``stdout=PIPE`` must be specified. - -* popen2 closes all file descriptors by default, but you have to specify - ``close_fds=True`` with :class:`Popen` to guarantee this behavior on - all platforms or past Python versions. - - Legacy Shell Invocation Functions --------------------------------- From e50fac96e82d857ecc024b4cd4e012493b077064 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 30 May 2024 07:48:18 -0700 Subject: [PATCH 278/903] gh-119336: Restore removed _PyLong_NumBits() function (#119418) It is used by the pywin32 project. --- Include/cpython/longobject.h | 9 +++++++++ Include/internal/pycore_long.h | 11 ----------- .../2024-05-22-17-50-48.gh-issue-119336.ff3qnS.rst | 1 + 3 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-05-22-17-50-48.gh-issue-119336.ff3qnS.rst diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 189229ee1035d8..96815938c8277a 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -60,6 +60,15 @@ PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); // There are no error cases. PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); +/* _PyLong_NumBits. Return the number of bits needed to represent the + absolute value of a long. For example, this returns 1 for 1 and -1, 2 + for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. + v must not be NULL, and must be a normalized long. + (size_t)-1 is returned and OverflowError set if the true result doesn't + fit in a size_t. +*/ +PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); + /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in base 256, and return a Python int with the same numeric value. If n is 0, the integer is 0. Else: diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index f04f66d053bab9..8513695c22e703 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -47,17 +47,6 @@ extern "C" { # error "_PY_LONG_DEFAULT_MAX_STR_DIGITS smaller than threshold." #endif -// _PyLong_NumBits. Return the number of bits needed to represent the -// absolute value of a long. For example, this returns 1 for 1 and -1, 2 -// for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. -// v must not be NULL, and must be a normalized long. -// (size_t)-1 is returned and OverflowError set if the true result doesn't -// fit in a size_t. -// -// Export for 'math' shared extension. -PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); - - /* runtime lifecycle */ extern PyStatus _PyLong_InitTypes(PyInterpreterState *); diff --git a/Misc/NEWS.d/next/C API/2024-05-22-17-50-48.gh-issue-119336.ff3qnS.rst b/Misc/NEWS.d/next/C API/2024-05-22-17-50-48.gh-issue-119336.ff3qnS.rst new file mode 100644 index 00000000000000..e530bb45d35e76 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-22-17-50-48.gh-issue-119336.ff3qnS.rst @@ -0,0 +1 @@ +Restore the removed ``_PyLong_NumBits()`` function. It is used by the pywin32 project. Patch by Ethan Smith From 6fb191be15fd49da10506de29b6393ffdf59b894 Mon Sep 17 00:00:00 2001 From: Awbert <119314310+SweetyAngel@users.noreply.github.com> Date: Thu, 30 May 2024 18:51:22 +0300 Subject: [PATCH 279/903] gh-119779: Fix pyporting howto docs (#119785) --- Doc/howto/pyporting.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index d560364107bd12..9f73c811cfcbc0 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -18,9 +18,9 @@ please see :ref:`cporting-howto`. The archived python-porting_ mailing list may contain some useful guidance. -Since Python 3.13 the original porting guide was discontinued. +Since Python 3.11 the original porting guide was discontinued. You can find the old guide in the -`archive `_. +`archive `_. Third-party guides From e91fc11fafb657cab88c5e6f13822432a3b9dc64 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 30 May 2024 17:38:37 +0100 Subject: [PATCH 280/903] gh-119786: create folder in cpython repo for internals documentation (#119787) --- InternalDocs/index.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 InternalDocs/index.md diff --git a/InternalDocs/index.md b/InternalDocs/index.md new file mode 100644 index 00000000000000..32b66a254bcf2c --- /dev/null +++ b/InternalDocs/index.md @@ -0,0 +1,12 @@ + +# CPython Internals Documentation + +The documentation in this folder is intended for CPython maintainers. +It describes implementation details of CPython, which should not be +assumed to be part of the Python language specification. These details +can change between any two CPython versions and should not be assumed +to hold for other implementations of the Python language. + +The core dev team attempts to keep this documentation up to date. If +it is not, please report that through the +[issue tracker](https://github.com/python/cpython/issues). From e875c2d752fed0a8d16958dc7b331e66a2476247 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 30 May 2024 20:22:52 +0300 Subject: [PATCH 281/903] gh-119791: Fix new Tkinter tests for wantobjects=0 (GH-119792) PhotoImage.get() retruns a string instead of a 3-tuple of integers in this case. --- Lib/test/test_tkinter/test_images.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index b8e549e314d27d..38371fe00d6eb5 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -581,13 +581,15 @@ def test_write(self): image.write(filename, background='#ff0000') image4 = tkinter.PhotoImage('::img::test4', master=self.root, format='ppm', file=filename) - self.assertEqual(image4.get(0, 0), (255, 0, 0)) + self.assertEqual(image4.get(0, 0), (255, 0, 0) if self.wantobjects else '255 0 0') self.assertEqual(image4.get(4, 6), image.get(4, 6)) image.write(filename, grayscale=True) image5 = tkinter.PhotoImage('::img::test5', master=self.root, format='ppm', file=filename) c = image5.get(4, 6) + if not self.wantobjects: + c = c.split() self.assertTrue(c[0] == c[1] == c[2], c) def test_data(self): @@ -597,7 +599,10 @@ def test_data(self): self.assertIsInstance(data, tuple) for row in data: self.assertIsInstance(row, str) - self.assertEqual(data[6].split()[4], '#%02x%02x%02x' % image.get(4, 6)) + c = image.get(4, 6) + if not self.wantobjects: + c = tuple(map(int, c.split())) + self.assertEqual(data[6].split()[4], '#%02x%02x%02x' % c) data = image.data('ppm') image2 = tkinter.PhotoImage('::img::test2', master=self.root, @@ -622,13 +627,15 @@ def test_data(self): data = image.data('ppm', background='#ff0000') image4 = tkinter.PhotoImage('::img::test4', master=self.root, format='ppm', data=data) - self.assertEqual(image4.get(0, 0), (255, 0, 0)) + self.assertEqual(image4.get(0, 0), (255, 0, 0) if self.wantobjects else '255 0 0') self.assertEqual(image4.get(4, 6), image.get(4, 6)) data = image.data('ppm', grayscale=True) image5 = tkinter.PhotoImage('::img::test5', master=self.root, format='ppm', data=data) c = image5.get(4, 6) + if not self.wantobjects: + c = c.split() self.assertTrue(c[0] == c[1] == c[2], c) From bf098d4157158e1e4b2ea78aba4ac82d72e24cff Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 30 May 2024 20:35:59 +0300 Subject: [PATCH 282/903] gh-109218: Refactor tests for the complex() constructor (GH-119635) * Share common classes. * Use exactly representable floats and exact tests. * Check the sign of zero components. * Remove duplicated tests (mostly left after merging int and long). * Reorder tests in more consistent way. * Test more error messages. * Add tests for missed cases. --- Lib/test/test_complex.py | 362 +++++++++++++++++++++------------------ 1 file changed, 195 insertions(+), 167 deletions(-) diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index fa3017b24e16c8..f29b7d3ebd31ab 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -5,7 +5,7 @@ INVALID_UNDERSCORE_LITERALS) from random import random -from math import atan2, isnan, copysign +from math import isnan, copysign import operator INF = float("inf") @@ -21,6 +21,27 @@ (1, 0+0j), ) +class WithIndex: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + +class WithFloat: + def __init__(self, value): + self.value = value + def __float__(self): + return self.value + +class ComplexSubclass(complex): + pass + +class WithComplex: + def __init__(self, value): + self.value = value + def __complex__(self): + return self.value + class ComplexTest(unittest.TestCase): def assertAlmostEqual(self, a, b): @@ -340,137 +361,90 @@ def test_conjugate(self): self.assertClose(complex(5.3, 9.8).conjugate(), 5.3-9.8j) def test_constructor(self): - class NS: - def __init__(self, value): self.value = value - def __complex__(self): return self.value - self.assertEqual(complex(NS(1+10j)), 1+10j) - self.assertRaises(TypeError, complex, NS(None)) - self.assertRaises(TypeError, complex, {}) - self.assertRaises(TypeError, complex, NS(1.5)) - self.assertRaises(TypeError, complex, NS(1)) - self.assertRaises(TypeError, complex, object()) - self.assertRaises(TypeError, complex, NS(4.25+0.5j), object()) - - self.assertAlmostEqual(complex("1+10j"), 1+10j) - self.assertAlmostEqual(complex(10), 10+0j) - self.assertAlmostEqual(complex(10.0), 10+0j) - self.assertAlmostEqual(complex(10), 10+0j) - self.assertAlmostEqual(complex(10+0j), 10+0j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10.0), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10.0), 1+10j) - self.assertAlmostEqual(complex(1.0,10), 1+10j) - self.assertAlmostEqual(complex(1.0,10), 1+10j) - self.assertAlmostEqual(complex(1.0,10.0), 1+10j) - self.assertAlmostEqual(complex(3.14+0j), 3.14+0j) - self.assertAlmostEqual(complex(3.14), 3.14+0j) - self.assertAlmostEqual(complex(314), 314.0+0j) - self.assertAlmostEqual(complex(314), 314.0+0j) - self.assertAlmostEqual(complex(3.14+0j, 0j), 3.14+0j) - self.assertAlmostEqual(complex(3.14, 0.0), 3.14+0j) - self.assertAlmostEqual(complex(314, 0), 314.0+0j) - self.assertAlmostEqual(complex(314, 0), 314.0+0j) - self.assertAlmostEqual(complex(0j, 3.14j), -3.14+0j) - self.assertAlmostEqual(complex(0.0, 3.14j), -3.14+0j) - self.assertAlmostEqual(complex(0j, 3.14), 3.14j) - self.assertAlmostEqual(complex(0.0, 3.14), 3.14j) - self.assertAlmostEqual(complex("1"), 1+0j) - self.assertAlmostEqual(complex("1j"), 1j) - self.assertAlmostEqual(complex(), 0) - self.assertAlmostEqual(complex("-1"), -1) - self.assertAlmostEqual(complex("+1"), +1) - self.assertAlmostEqual(complex("(1+2j)"), 1+2j) - self.assertAlmostEqual(complex("(1.3+2.2j)"), 1.3+2.2j) - self.assertAlmostEqual(complex("3.14+1J"), 3.14+1j) - self.assertAlmostEqual(complex(" ( +3.14-6J )"), 3.14-6j) - self.assertAlmostEqual(complex(" ( +3.14-J )"), 3.14-1j) - self.assertAlmostEqual(complex(" ( +3.14+j )"), 3.14+1j) - self.assertAlmostEqual(complex("J"), 1j) - self.assertAlmostEqual(complex("( j )"), 1j) - self.assertAlmostEqual(complex("+J"), 1j) - self.assertAlmostEqual(complex("( -j)"), -1j) - self.assertAlmostEqual(complex('1e-500'), 0.0 + 0.0j) - self.assertAlmostEqual(complex('-1e-500j'), 0.0 - 0.0j) - self.assertAlmostEqual(complex('-1e-500+1e-500j'), -0.0 + 0.0j) - self.assertEqual(complex('1-1j'), 1.0 - 1j) - self.assertEqual(complex('1J'), 1j) - - class complex2(complex): pass - self.assertAlmostEqual(complex(complex2(1+1j)), 1+1j) - self.assertAlmostEqual(complex(real=17, imag=23), 17+23j) - self.assertAlmostEqual(complex(real=17+23j), 17+23j) - self.assertAlmostEqual(complex(real=17+23j, imag=23), 17+46j) - self.assertAlmostEqual(complex(real=1+2j, imag=3+4j), -3+5j) + def check(z, x, y): + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + + check(complex(), 0.0, 0.0) + check(complex(10), 10.0, 0.0) + check(complex(4.25), 4.25, 0.0) + check(complex(4.25+0j), 4.25, 0.0) + check(complex(4.25+0.5j), 4.25, 0.5) + check(complex(ComplexSubclass(4.25+0.5j)), 4.25, 0.5) + check(complex(WithComplex(4.25+0.5j)), 4.25, 0.5) + + check(complex(1, 10), 1.0, 10.0) + check(complex(1, 10.0), 1.0, 10.0) + check(complex(1, 4.25), 1.0, 4.25) + check(complex(1.0, 10), 1.0, 10.0) + check(complex(4.25, 10), 4.25, 10.0) + check(complex(1.0, 10.0), 1.0, 10.0) + check(complex(4.25, 0.5), 4.25, 0.5) + + check(complex(4.25+0j, 0), 4.25, 0.0) + check(complex(ComplexSubclass(4.25+0j), 0), 4.25, 0.0) + check(complex(WithComplex(4.25+0j), 0), 4.25, 0.0) + check(complex(4.25j, 0), 0.0, 4.25) + check(complex(0j, 4.25), 0.0, 4.25) + check(complex(0, 4.25+0j), 0.0, 4.25) + check(complex(0, ComplexSubclass(4.25+0j)), 0.0, 4.25) + with self.assertRaisesRegex(TypeError, + "second argument must be a number, not 'WithComplex'"): + complex(0, WithComplex(4.25+0j)) + check(complex(0.0, 4.25j), -4.25, 0.0) + check(complex(4.25+0j, 0j), 4.25, 0.0) + check(complex(4.25j, 0j), 0.0, 4.25) + check(complex(0j, 4.25+0j), 0.0, 4.25) + check(complex(0j, 4.25j), -4.25, 0.0) + + check(complex(real=4.25), 4.25, 0.0) + check(complex(real=4.25+0j), 4.25, 0.0) + check(complex(real=4.25+1.5j), 4.25, 1.5) + check(complex(imag=1.5), 0.0, 1.5) + check(complex(real=4.25, imag=1.5), 4.25, 1.5) + check(complex(4.25, imag=1.5), 4.25, 1.5) # check that the sign of a zero in the real or imaginary part - # is preserved when constructing from two floats. (These checks - # are harmless on systems without support for signed zeros.) - def split_zeros(x): - """Function that produces different results for 0. and -0.""" - return atan2(x, -1.) - - self.assertEqual(split_zeros(complex(1., 0.).imag), split_zeros(0.)) - self.assertEqual(split_zeros(complex(1., -0.).imag), split_zeros(-0.)) - self.assertEqual(split_zeros(complex(0., 1.).real), split_zeros(0.)) - self.assertEqual(split_zeros(complex(-0., 1.).real), split_zeros(-0.)) - - c = 3.14 + 1j - self.assertTrue(complex(c) is c) - del c - - self.assertRaises(TypeError, complex, "1", "1") - self.assertRaises(TypeError, complex, 1, "1") - - # SF bug 543840: complex(string) accepts strings with \0 - # Fixed in 2.3. - self.assertRaises(ValueError, complex, '1+1j\0j') - - self.assertRaises(TypeError, int, 5+3j) - self.assertRaises(TypeError, int, 5+3j) - self.assertRaises(TypeError, float, 5+3j) - self.assertRaises(ValueError, complex, "") - self.assertRaises(TypeError, complex, None) - self.assertRaisesRegex(TypeError, "not 'NoneType'", complex, None) - self.assertRaises(ValueError, complex, "\0") - self.assertRaises(ValueError, complex, "3\09") - self.assertRaises(TypeError, complex, "1", "2") - self.assertRaises(TypeError, complex, "1", 42) - self.assertRaises(TypeError, complex, 1, "2") - self.assertRaises(ValueError, complex, "1+") - self.assertRaises(ValueError, complex, "1+1j+1j") - self.assertRaises(ValueError, complex, "--") - self.assertRaises(ValueError, complex, "(1+2j") - self.assertRaises(ValueError, complex, "1+2j)") - self.assertRaises(ValueError, complex, "1+(2j)") - self.assertRaises(ValueError, complex, "(1+2j)123") - self.assertRaises(ValueError, complex, "x") - self.assertRaises(ValueError, complex, "1j+2") - self.assertRaises(ValueError, complex, "1e1ej") - self.assertRaises(ValueError, complex, "1e++1ej") - self.assertRaises(ValueError, complex, ")1+2j(") - self.assertRaisesRegex( - TypeError, + # is preserved when constructing from two floats. + for x in 1.0, -1.0: + for y in 0.0, -0.0: + check(complex(x, y), x, y) + check(complex(y, x), y, x) + + c = complex(4.25, 1.5) + self.assertIs(complex(c), c) + c2 = ComplexSubclass(c) + self.assertEqual(c2, c) + self.assertIs(type(c2), ComplexSubclass) + del c, c2 + + self.assertRaisesRegex(TypeError, + "first argument must be a string or a number, not 'dict'", + complex, {}) + self.assertRaisesRegex(TypeError, + "first argument must be a string or a number, not 'NoneType'", + complex, None) + self.assertRaisesRegex(TypeError, "first argument must be a string or a number, not 'dict'", - complex, {1:2}, 1) - self.assertRaisesRegex( - TypeError, + complex, {1:2}, 0) + self.assertRaisesRegex(TypeError, + "can't take second arg if first is a string", + complex, '1', 0) + self.assertRaisesRegex(TypeError, "second argument must be a number, not 'dict'", - complex, 1, {1:2}) - # the following three are accepted by Python 2.6 - self.assertRaises(ValueError, complex, "1..1j") - self.assertRaises(ValueError, complex, "1.11.1j") - self.assertRaises(ValueError, complex, "1e1.1j") - - # check that complex accepts long unicode strings - self.assertEqual(type(complex("1"*500)), complex) - # check whitespace processing - self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j) - # Invalid unicode string - # See bpo-34087 - self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f') + complex, 0, {1:2}) + self.assertRaisesRegex(TypeError, + "second arg can't be a string", + complex, 0, '1') + + self.assertRaises(TypeError, complex, WithComplex(1.5)) + self.assertRaises(TypeError, complex, WithComplex(1)) + self.assertRaises(TypeError, complex, WithComplex(None)) + self.assertRaises(TypeError, complex, WithComplex(4.25+0j), object()) + self.assertRaises(TypeError, complex, WithComplex(1.5), object()) + self.assertRaises(TypeError, complex, WithComplex(1), object()) + self.assertRaises(TypeError, complex, WithComplex(None), object()) class EvilExc(Exception): pass @@ -481,33 +455,33 @@ def __complex__(self): self.assertRaises(EvilExc, complex, evilcomplex()) - class float2: - def __init__(self, value): - self.value = value - def __float__(self): - return self.value - - self.assertAlmostEqual(complex(float2(42.)), 42) - self.assertAlmostEqual(complex(real=float2(17.), imag=float2(23.)), 17+23j) - self.assertRaises(TypeError, complex, float2(None)) - - class MyIndex: - def __init__(self, value): - self.value = value - def __index__(self): - return self.value - - self.assertAlmostEqual(complex(MyIndex(42)), 42.0+0.0j) - self.assertAlmostEqual(complex(123, MyIndex(42)), 123.0+42.0j) - self.assertRaises(OverflowError, complex, MyIndex(2**2000)) - self.assertRaises(OverflowError, complex, 123, MyIndex(2**2000)) + check(complex(WithFloat(4.25)), 4.25, 0.0) + check(complex(WithFloat(4.25), 1.5), 4.25, 1.5) + check(complex(1.5, WithFloat(4.25)), 1.5, 4.25) + self.assertRaises(TypeError, complex, WithFloat(42)) + self.assertRaises(TypeError, complex, WithFloat(42), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithFloat(42)) + self.assertRaises(TypeError, complex, WithFloat(None)) + self.assertRaises(TypeError, complex, WithFloat(None), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithFloat(None)) + + check(complex(WithIndex(42)), 42.0, 0.0) + check(complex(WithIndex(42), 1.5), 42.0, 1.5) + check(complex(1.5, WithIndex(42)), 1.5, 42.0) + self.assertRaises(OverflowError, complex, WithIndex(2**2000)) + self.assertRaises(OverflowError, complex, WithIndex(2**2000), 1.5) + self.assertRaises(OverflowError, complex, 1.5, WithIndex(2**2000)) + self.assertRaises(TypeError, complex, WithIndex(None)) + self.assertRaises(TypeError, complex, WithIndex(None), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithIndex(None)) class MyInt: def __int__(self): return 42 self.assertRaises(TypeError, complex, MyInt()) - self.assertRaises(TypeError, complex, 123, MyInt()) + self.assertRaises(TypeError, complex, MyInt(), 1.5) + self.assertRaises(TypeError, complex, 1.5, MyInt()) class complex0(complex): """Test usage of __complex__() when inheriting from 'complex'""" @@ -527,9 +501,9 @@ class complex2(complex): def __complex__(self): return None - self.assertEqual(complex(complex0(1j)), 42j) + check(complex(complex0(1j)), 0.0, 42.0) with self.assertWarns(DeprecationWarning): - self.assertEqual(complex(complex1(1j)), 2j) + check(complex(complex1(1j)), 0.0, 2.0) self.assertRaises(TypeError, complex, complex2(1j)) def test___complex__(self): @@ -537,36 +511,93 @@ def test___complex__(self): self.assertEqual(z.__complex__(), z) self.assertEqual(type(z.__complex__()), complex) - class complex_subclass(complex): - pass - - z = complex_subclass(3 + 4j) + z = ComplexSubclass(3 + 4j) self.assertEqual(z.__complex__(), 3 + 4j) self.assertEqual(type(z.__complex__()), complex) @support.requires_IEEE_754 def test_constructor_special_numbers(self): - class complex2(complex): - pass for x in 0.0, -0.0, INF, -INF, NAN: for y in 0.0, -0.0, INF, -INF, NAN: with self.subTest(x=x, y=y): z = complex(x, y) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex2(x, y) - self.assertIs(type(z), complex2) + z = ComplexSubclass(x, y) + self.assertIs(type(z), ComplexSubclass) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex(complex2(x, y)) + z = complex(ComplexSubclass(x, y)) self.assertIs(type(z), complex) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex2(complex(x, y)) - self.assertIs(type(z), complex2) + z = ComplexSubclass(complex(x, y)) + self.assertIs(type(z), ComplexSubclass) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) + def test_constructor_from_string(self): + def check(z, x, y): + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + + check(complex("1"), 1.0, 0.0) + check(complex("1j"), 0.0, 1.0) + check(complex("-1"), -1.0, 0.0) + check(complex("+1"), 1.0, 0.0) + check(complex("1+2j"), 1.0, 2.0) + check(complex("(1+2j)"), 1.0, 2.0) + check(complex("(1.5+4.25j)"), 1.5, 4.25) + check(complex("4.25+1J"), 4.25, 1.0) + check(complex(" ( +4.25-6J )"), 4.25, -6.0) + check(complex(" ( +4.25-J )"), 4.25, -1.0) + check(complex(" ( +4.25+j )"), 4.25, 1.0) + check(complex("J"), 0.0, 1.0) + check(complex("( j )"), 0.0, 1.0) + check(complex("+J"), 0.0, 1.0) + check(complex("( -j)"), 0.0, -1.0) + check(complex('1-1j'), 1.0, -1.0) + check(complex('1J'), 0.0, 1.0) + + check(complex('1e-500'), 0.0, 0.0) + check(complex('-1e-500j'), 0.0, -0.0) + check(complex('1e-500+1e-500j'), 0.0, 0.0) + check(complex('-1e-500+1e-500j'), -0.0, 0.0) + check(complex('1e-500-1e-500j'), 0.0, -0.0) + check(complex('-1e-500-1e-500j'), -0.0, -0.0) + + # SF bug 543840: complex(string) accepts strings with \0 + # Fixed in 2.3. + self.assertRaises(ValueError, complex, '1+1j\0j') + self.assertRaises(ValueError, complex, "") + self.assertRaises(ValueError, complex, "\0") + self.assertRaises(ValueError, complex, "3\09") + self.assertRaises(ValueError, complex, "1+") + self.assertRaises(ValueError, complex, "1+1j+1j") + self.assertRaises(ValueError, complex, "--") + self.assertRaises(ValueError, complex, "(1+2j") + self.assertRaises(ValueError, complex, "1+2j)") + self.assertRaises(ValueError, complex, "1+(2j)") + self.assertRaises(ValueError, complex, "(1+2j)123") + self.assertRaises(ValueError, complex, "x") + self.assertRaises(ValueError, complex, "1j+2") + self.assertRaises(ValueError, complex, "1e1ej") + self.assertRaises(ValueError, complex, "1e++1ej") + self.assertRaises(ValueError, complex, ")1+2j(") + # the following three are accepted by Python 2.6 + self.assertRaises(ValueError, complex, "1..1j") + self.assertRaises(ValueError, complex, "1.11.1j") + self.assertRaises(ValueError, complex, "1e1.1j") + + # check that complex accepts long unicode strings + self.assertIs(type(complex("1"*500)), complex) + # check whitespace processing + self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j) + # Invalid unicode string + # See bpo-34087 + self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f') + def test_constructor_negative_nans_from_string(self): self.assertEqual(copysign(1., complex("-nan").real), -1.) self.assertEqual(copysign(1., complex("-nanj").imag), -1.) @@ -645,9 +676,6 @@ def test(v, expected, test_fn=self.assertEqual): test(complex(-0., -0.), "(-0-0j)") def test_pos(self): - class ComplexSubclass(complex): - pass - self.assertEqual(+(1+6j), 1+6j) self.assertEqual(+ComplexSubclass(1, 6), 1+6j) self.assertIs(type(+ComplexSubclass(1, 6)), complex) @@ -667,8 +695,8 @@ def test_getnewargs(self): def test_plus_minus_0j(self): # test that -0j and 0j literals are not identified z1, z2 = 0j, -0j - self.assertEqual(atan2(z1.imag, -1.), atan2(0., -1.)) - self.assertEqual(atan2(z2.imag, -1.), atan2(-0., -1.)) + self.assertFloatsAreIdentical(z1.imag, 0.0) + self.assertFloatsAreIdentical(z2.imag, -0.0) @support.requires_IEEE_754 def test_negated_imaginary_literal(self): From 1c04c63ced5038e8f45a2aac7dc45f0815a4ddc5 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 30 May 2024 13:48:28 -0400 Subject: [PATCH 283/903] gh-119729: Use 't' in pkg-config file name for free-threaded build (#119738) For example, the free-threaded build now generates `lib/pkgconfig/python-3.13t.pc` and the debug build generates `lib/pkgconfig/python-3.13d.pc`. --- Doc/whatsnew/3.13.rst | 4 ++++ Makefile.pre.in | 14 +++++++------- .../2024-05-29-17-40-50.gh-issue-119729.k0xJ5U.rst | 5 +++++ 3 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-05-29-17-40-50.gh-issue-119729.k0xJ5U.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 2b1b5fdb974b26..bcded65eba3103 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -2181,6 +2181,10 @@ Build Changes The bundled mimalloc has custom changes, see :gh:`113141` for details. (Contributed by Dino Viehland in :gh:`109914`.) +* On POSIX systems, the pkg-config (``.pc``) filenames now include the ABI + flags. For example, the free-threaded build generates ``python-3.13t.pc`` + and the debug build generates ``python-3.13d.pc``. + Porting to Python 3.13 ====================== diff --git a/Makefile.pre.in b/Makefile.pre.in index 08918405403127..a3fca80d4448ca 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -40,6 +40,7 @@ LINKCC= @LINKCC@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ +ABIFLAGS= @ABIFLAGS@ LDVERSION= @LDVERSION@ MODULE_LDFLAGS=@MODULE_LDFLAGS@ GITVERSION= @GITVERSION@ @@ -150,7 +151,6 @@ INCLUDEDIR= @includedir@ CONFINCLUDEDIR= $(exec_prefix)/include PLATLIBDIR= @PLATLIBDIR@ SCRIPTDIR= $(prefix)/$(PLATLIBDIR) -ABIFLAGS= @ABIFLAGS@ # executable name for shebangs EXENAME= $(BINDIR)/python$(LDVERSION)$(EXE) # Variable used by ensurepip @@ -2262,10 +2262,10 @@ bininstall: commoninstall altbininstall -if test "$(VERSION)" != "$(LDVERSION)"; then \ rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)-config; \ (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(LDVERSION)-config python$(VERSION)-config); \ - rm -f $(DESTDIR)$(LIBPC)/python-$(LDVERSION).pc; \ - (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION).pc python-$(LDVERSION).pc); \ - rm -f $(DESTDIR)$(LIBPC)/python-$(LDVERSION)-embed.pc; \ - (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION)-embed.pc python-$(LDVERSION)-embed.pc); \ + rm -f $(DESTDIR)$(LIBPC)/python-$(VERSION).pc; \ + (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(LDVERSION).pc python-$(VERSION).pc); \ + rm -f $(DESTDIR)$(LIBPC)/python-$(VERSION)-embed.pc; \ + (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(LDVERSION)-embed.pc python-$(VERSION)-embed.pc); \ fi -rm -f $(DESTDIR)$(BINDIR)/python3-config (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-config python3-config) @@ -2702,8 +2702,8 @@ libainstall: all scripts $(INSTALL_DATA) Modules/Setup.bootstrap $(DESTDIR)$(LIBPL)/Setup.bootstrap $(INSTALL_DATA) Modules/Setup.stdlib $(DESTDIR)$(LIBPL)/Setup.stdlib $(INSTALL_DATA) Modules/Setup.local $(DESTDIR)$(LIBPL)/Setup.local - $(INSTALL_DATA) Misc/python.pc $(DESTDIR)$(LIBPC)/python-$(VERSION).pc - $(INSTALL_DATA) Misc/python-embed.pc $(DESTDIR)$(LIBPC)/python-$(VERSION)-embed.pc + $(INSTALL_DATA) Misc/python.pc $(DESTDIR)$(LIBPC)/python-$(LDVERSION).pc + $(INSTALL_DATA) Misc/python-embed.pc $(DESTDIR)$(LIBPC)/python-$(LDVERSION)-embed.pc $(INSTALL_SCRIPT) $(srcdir)/Modules/makesetup $(DESTDIR)$(LIBPL)/makesetup $(INSTALL_SCRIPT) $(srcdir)/install-sh $(DESTDIR)$(LIBPL)/install-sh $(INSTALL_SCRIPT) python-config.py $(DESTDIR)$(LIBPL)/python-config.py diff --git a/Misc/NEWS.d/next/Build/2024-05-29-17-40-50.gh-issue-119729.k0xJ5U.rst b/Misc/NEWS.d/next/Build/2024-05-29-17-40-50.gh-issue-119729.k0xJ5U.rst new file mode 100644 index 00000000000000..7ac300ecf40326 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-29-17-40-50.gh-issue-119729.k0xJ5U.rst @@ -0,0 +1,5 @@ +On POSIX systems, the pkg-config (``.pc``) filenames now include the ABI +flags, which may include debug ("d") and free-threaded ("t"). For example: +* ``python-3.14.pc`` (default, non-debug build) +* ``python-3.14d.pc`` (default, debug build) +* ``python-3.14t.pc`` (free-threaded build) From ec1ba264607b2b7b98d2602f5536a1d02981efc6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 30 May 2024 23:20:07 +0300 Subject: [PATCH 284/903] gh-109218: Improve documentation for the complex() constructor (GH-119687) * Remove the equivalence with real+imag*1j which can be incorrect in corner cases (non-finite numbers, the sign of zeroes). * Separately document the three roles of the constructor: parsing a string, converting a number, and constructing a complex from components. * Document positional-only parameters of complex(), float(), int() and bool() as positional-only. * Add examples for complex() and int(). * Specify the grammar of the string for complex(). * Improve the grammar of the string for float(). * Describe more explicitly the behavior when real and/or imag arguments are complex numbers. (This will be deprecated in future.) --- Doc/library/cmath.rst | 7 +- Doc/library/functions.rst | 173 +++++++++++++++++++++---------- Objects/clinic/complexobject.c.h | 9 +- Objects/complexobject.c | 9 +- 4 files changed, 135 insertions(+), 63 deletions(-) diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 65e98e09ad7ae3..381a8332f4b187 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -43,10 +43,7 @@ Conversions to and from polar coordinates A Python complex number ``z`` is stored internally using *rectangular* or *Cartesian* coordinates. It is completely determined by its *real -part* ``z.real`` and its *imaginary part* ``z.imag``. In other -words:: - - z == z.real + z.imag*1j +part* ``z.real`` and its *imaginary part* ``z.imag``. *Polar coordinates* give an alternative way to represent a complex number. In polar coordinates, a complex number *z* is defined by the @@ -90,7 +87,7 @@ rectangular coordinates to polar coordinates and back. .. function:: rect(r, phi) Return the complex number *x* with polar coordinates *r* and *phi*. - Equivalent to ``r * (math.cos(phi) + math.sin(phi)*1j)``. + Equivalent to ``complex(r * math.cos(phi), r * math.sin(phi))``. Power and logarithmic functions diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index cb9b650badcfbd..c07b1043afe627 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -141,10 +141,11 @@ are always available. They are listed here in alphabetical order. See also :func:`format` for more information. -.. class:: bool(x=False) +.. class:: bool(object=False, /) - Return a Boolean value, i.e. one of ``True`` or ``False``. *x* is converted - using the standard :ref:`truth testing procedure `. If *x* is false + Return a Boolean value, i.e. one of ``True`` or ``False``. The argument + is converted using the standard :ref:`truth testing procedure `. + If the argument is false or omitted, this returns ``False``; otherwise, it returns ``True``. The :class:`bool` class is a subclass of :class:`int` (see :ref:`typesnumeric`). It cannot be subclassed further. Its only instances are ``False`` and @@ -153,7 +154,7 @@ are always available. They are listed here in alphabetical order. .. index:: pair: Boolean; type .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The parameter is now positional-only. .. function:: breakpoint(*args, **kws) @@ -371,29 +372,73 @@ are always available. They are listed here in alphabetical order. support for top-level ``await``, ``async for``, and ``async with``. -.. class:: complex(real=0, imag=0) - complex(string) +.. class:: complex(number=0, /) + complex(string, /) + complex(real=0, imag=0) + + Convert a single string or number to a complex number, or create a + complex number from real and imaginary parts. + + Examples: + + .. doctest:: + + >>> complex('+1.23') + (1.23+0j) + >>> complex('-4.5j') + -4.5j + >>> complex('-1.23+4.5j') + (-1.23+4.5j) + >>> complex('\t( -1.23+4.5J )\n') + (-1.23+4.5j) + >>> complex('-Infinity+NaNj') + (-inf+nanj) + >>> complex(1.23) + (1.23+0j) + >>> complex(imag=-4.5) + -4.5j + >>> complex(-1.23, 4.5) + (-1.23+4.5j) + + If the argument is a string, it must contain either a real part (in the + same format as for :func:`float`) or an imaginary part (in the same + format but with a ``'j'`` or ``'J'`` suffix), or both real and imaginary + parts (the sign of the imaginary part is mandatory in this case). + The string can optionally be surrounded by whitespaces and the round + parentheses ``'('`` and ``')'``, which are ignored. + The string must not contain whitespace between ``'+'``, ``'-'``, the + ``'j'`` or ``'J'`` suffix, and the decimal number. + For example, ``complex('1+2j')`` is fine, but ``complex('1 + 2j')`` raises + :exc:`ValueError`. + More precisely, the input must conform to the :token:`~float:complexvalue` + production rule in the following grammar, after parentheses and leading and + trailing whitespace characters are removed: - Return a complex number with the value *real* + *imag*\*1j or convert a string - or number to a complex number. If the first parameter is a string, it will - be interpreted as a complex number and the function must be called without a - second parameter. The second parameter can never be a string. Each argument - may be any numeric type (including complex). If *imag* is omitted, it - defaults to zero and the constructor serves as a numeric conversion like - :class:`int` and :class:`float`. If both arguments are omitted, returns - ``0j``. + .. productionlist:: float + complexvalue: `floatvalue` | + : `floatvalue` ("j" | "J") | + : `floatvalue` `sign` `absfloatvalue` ("j" | "J") + If the argument is a number, the constructor serves as a numeric + conversion like :class:`int` and :class:`float`. For a general Python object ``x``, ``complex(x)`` delegates to - ``x.__complex__()``. If :meth:`~object.__complex__` is not defined then it falls back - to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + ``x.__complex__()``. + If :meth:`~object.__complex__` is not defined then it falls back + to :meth:`~object.__float__`. + If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. - .. note:: + If two arguments are provided or keyword arguments are used, each argument + may be any numeric type (including complex). + If both arguments are real numbers, return a complex number with the real + component *real* and the imaginary component *imag*. + If both arguments are complex numbers, return a complex number with the real + component ``real.real-imag.imag`` and the imaginary component + ``real.imag+imag.real``. + If one of arguments is a real number, only its real component is used in + the above expressions. - When converting from a string, the string must not contain whitespace - around the central ``+`` or ``-`` operator. For example, - ``complex('1+2j')`` is fine, but ``complex('1 + 2j')`` raises - :exc:`ValueError`. + If all arguments are omitted, returns ``0j``. The complex type is described in :ref:`typesnumeric`. @@ -682,21 +727,38 @@ are always available. They are listed here in alphabetical order. elements of *iterable* for which *function* is false. -.. class:: float(x=0.0) +.. class:: float(number=0.0, /) + float(string, /) .. index:: single: NaN single: Infinity - Return a floating point number constructed from a number or string *x*. + Return a floating point number constructed from a number or a string. + + Examples: + + .. doctest:: + + >>> float('+1.23') + 1.23 + >>> float(' -12345\n') + -12345.0 + >>> float('1e-003') + 0.001 + >>> float('+1E6') + 1000000.0 + >>> float('-Infinity') + -inf If the argument is a string, it should contain a decimal number, optionally preceded by a sign, and optionally embedded in whitespace. The optional sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value produced. The argument may also be a string representing a NaN - (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the ``floatvalue`` production rule in the following - grammar, after leading and trailing whitespace characters are removed: + (not-a-number), or positive or negative infinity. + More precisely, the input must conform to the :token:`~float:floatvalue` + production rule in the following grammar, after leading and trailing + whitespace characters are removed: .. productionlist:: float sign: "+" | "-" @@ -705,9 +767,10 @@ are always available. They are listed here in alphabetical order. digit: digitpart: `digit` (["_"] `digit`)* number: [`digitpart`] "." `digitpart` | `digitpart` ["."] - exponent: ("e" | "E") ["+" | "-"] `digitpart` - floatnumber: number [`exponent`] - floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) + exponent: ("e" | "E") [`sign`] `digitpart` + floatnumber: `number` [`exponent`] + absfloatvalue: `floatnumber` | `infinity` | `nan` + floatvalue: [`sign`] `absfloatvalue` Case is not significant, so, for example, "inf", "Inf", "INFINITY", and "iNfINity" are all acceptable spellings for positive infinity. @@ -723,26 +786,13 @@ are always available. They are listed here in alphabetical order. If no argument is given, ``0.0`` is returned. - Examples:: - - >>> float('+1.23') - 1.23 - >>> float(' -12345\n') - -12345.0 - >>> float('1e-003') - 0.001 - >>> float('+1E6') - 1000000.0 - >>> float('-Infinity') - -inf - The float type is described in :ref:`typesnumeric`. .. versionchanged:: 3.6 Grouping digits with underscores as in code literals is allowed. .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The parameter is now positional-only. .. versionchanged:: 3.8 Falls back to :meth:`~object.__index__` if :meth:`~object.__float__` is not defined. @@ -926,17 +976,36 @@ are always available. They are listed here in alphabetical order. with the result after successfully reading input. -.. class:: int(x=0) - int(x, base=10) +.. class:: int(number=0, /) + int(string, /, base=10) + + Return an integer object constructed from a number or a string, or return + ``0`` if no arguments are given. + + Examples: + + .. doctest:: + + >>> int(123.45) + 123 + >>> int('123') + 123 + >>> int(' -12_345\n') + -12345 + >>> int('FACE', 16) + 64206 + >>> int('0xface', 0) + 64206 + >>> int('01110011', base=2) + 115 - Return an integer object constructed from a number or string *x*, or return - ``0`` if no arguments are given. If *x* defines :meth:`~object.__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`~object.__index__`, - it returns ``x.__index__()``. If *x* defines :meth:`~object.__trunc__`, + If the argument defines :meth:`~object.__int__`, + ``int(x)`` returns ``x.__int__()``. If the argument defines :meth:`~object.__index__`, + it returns ``x.__index__()``. If the argument defines :meth:`~object.__trunc__`, it returns ``x.__trunc__()``. For floating point numbers, this truncates towards zero. - If *x* is not a number or if *base* is given, then *x* must be a string, + If the argument is not a number or if *base* is given, then it must be a string, :class:`bytes`, or :class:`bytearray` instance representing an integer in radix *base*. Optionally, the string can be preceded by ``+`` or ``-`` (with no space in between), have leading zeros, be surrounded by whitespace, @@ -966,7 +1035,7 @@ are always available. They are listed here in alphabetical order. Grouping digits with underscores as in code literals is allowed. .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The first parameter is now positional-only. .. versionchanged:: 3.8 Falls back to :meth:`~object.__index__` if :meth:`~object.__int__` is not defined. @@ -977,7 +1046,7 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.11 :class:`int` string inputs and string representations can be limited to help avoid denial of service attacks. A :exc:`ValueError` is raised when - the limit is exceeded while converting a string *x* to an :class:`int` or + the limit is exceeded while converting a string to an :class:`int` or when converting an :class:`int` into a string would exceed the limit. See the :ref:`integer string conversion length limitation ` documentation. diff --git a/Objects/clinic/complexobject.c.h b/Objects/clinic/complexobject.c.h index 49b50304021f7b..46c3b352562445 100644 --- a/Objects/clinic/complexobject.c.h +++ b/Objects/clinic/complexobject.c.h @@ -94,9 +94,12 @@ PyDoc_STRVAR(complex_new__doc__, "complex(real=0, imag=0)\n" "--\n" "\n" -"Create a complex number from a real part and an optional imaginary part.\n" +"Create a complex number from a string or numbers.\n" "\n" -"This is equivalent to (real + imag*1j) where imag defaults to 0."); +"If a string is given, parse it as a complex number.\n" +"If a single number is given, convert it to a complex number.\n" +"If the \'real\' or \'imag\' arguments are given, create a complex number\n" +"with the specified real and imaginary components."); static PyObject * complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i); @@ -157,4 +160,4 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=04e6261649967b30 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=295ecfd71389d7fe input=a9049054013a1b77]*/ diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 943c5ccabfd5c4..17ee43725dd70d 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -911,14 +911,17 @@ complex.__new__ as complex_new real as r: object(c_default="NULL") = 0 imag as i: object(c_default="NULL") = 0 -Create a complex number from a real part and an optional imaginary part. +Create a complex number from a string or numbers. -This is equivalent to (real + imag*1j) where imag defaults to 0. +If a string is given, parse it as a complex number. +If a single number is given, convert it to a complex number. +If the 'real' or 'imag' arguments are given, create a complex number +with the specified real and imaginary components. [clinic start generated code]*/ static PyObject * complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) -/*[clinic end generated code: output=b6c7dd577b537dc1 input=f4c667f2596d4fd1]*/ +/*[clinic end generated code: output=b6c7dd577b537dc1 input=ff4268dc540958a4]*/ { PyObject *tmp; PyNumberMethods *nbr, *nbi = NULL; From deda85717b2557c6bad8b9a51719c60ac510818f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 30 May 2024 23:26:46 +0300 Subject: [PATCH 285/903] Docs: `shutil.rmtree`'s `onerror` has no pending removal version (#118947) --- Doc/whatsnew/3.13.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index bcded65eba3103..241c07e781af1f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1676,9 +1676,6 @@ Pending Removal in Python 3.14 * ``master_open()``: use :func:`pty.openpty`. * ``slave_open()``: use :func:`pty.openpty`. -* :func:`shutil.rmtree` *onerror* parameter is deprecated in 3.12, - and will be removed in 3.14: use the *onexc* parameter instead. - * :mod:`sqlite3`: * :data:`!version` and :data:`!version_info`. @@ -1854,6 +1851,9 @@ although there is currently no date scheduled for their removal. * :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. +* :mod:`shutil`: :func:`~shutil.rmtree`'s *onerror* parameter is deprecated in + Python 3.12; use the *onexc* parameter instead. + * :mod:`ssl` options and protocols: * :class:`ssl.SSLContext` without protocol argument is deprecated. From ef01e95ae3659015c2ebe4ecdc048aadcda89930 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 30 May 2024 23:30:57 +0300 Subject: [PATCH 286/903] gh-109218: Deprecate weird cases in the complex() constructor (GH-119620) * Passing a string as the "real" keyword argument is now an error; it should only be passed as a single positional argument. * Passing a complex number as the "real" or "imag" argument is now deprecated; it should only be passed as a single positional argument. --- Doc/library/functions.rst | 4 + Doc/whatsnew/3.14.rst | 4 + Lib/test/test_complex.py | 70 ++++++--- Lib/test/test_fractions.py | 5 +- ...-05-27-19-13-49.gh-issue-109218.-sdDg0.rst | 3 + Objects/complexobject.c | 139 +++++++++++++----- 6 files changed, 164 insertions(+), 61 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-27-19-13-49.gh-issue-109218.-sdDg0.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index c07b1043afe627..7291461c69acd2 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -449,6 +449,10 @@ are always available. They are listed here in alphabetical order. Falls back to :meth:`~object.__index__` if :meth:`~object.__complex__` and :meth:`~object.__float__` are not defined. + .. deprecated:: 3.14 + Passing a complex number as the *real* or *imag* argument is now + deprecated; it should only be passed as a single positional argument. + .. function:: delattr(object, name) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 8c37825430c2cf..d443cf9bc56b98 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -103,6 +103,10 @@ Optimizations Deprecated ========== +* Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; it should only be passed + as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) Removed diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index f29b7d3ebd31ab..fb510ca9b70902 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -382,25 +382,53 @@ def check(z, x, y): check(complex(1.0, 10.0), 1.0, 10.0) check(complex(4.25, 0.5), 4.25, 0.5) - check(complex(4.25+0j, 0), 4.25, 0.0) - check(complex(ComplexSubclass(4.25+0j), 0), 4.25, 0.0) - check(complex(WithComplex(4.25+0j), 0), 4.25, 0.0) - check(complex(4.25j, 0), 0.0, 4.25) - check(complex(0j, 4.25), 0.0, 4.25) - check(complex(0, 4.25+0j), 0.0, 4.25) - check(complex(0, ComplexSubclass(4.25+0j)), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25+0j, 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not .*ComplexSubclass"): + check(complex(ComplexSubclass(4.25+0j), 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not .*WithComplex"): + check(complex(WithComplex(4.25+0j), 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25j, 0), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not complex"): + check(complex(0, 4.25+0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not .*ComplexSubclass"): + check(complex(0, ComplexSubclass(4.25+0j)), 0.0, 4.25) with self.assertRaisesRegex(TypeError, - "second argument must be a number, not 'WithComplex'"): + "argument 'imag' must be a real number, not .*WithComplex"): complex(0, WithComplex(4.25+0j)) - check(complex(0.0, 4.25j), -4.25, 0.0) - check(complex(4.25+0j, 0j), 4.25, 0.0) - check(complex(4.25j, 0j), 0.0, 4.25) - check(complex(0j, 4.25+0j), 0.0, 4.25) - check(complex(0j, 4.25j), -4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not complex"): + check(complex(0.0, 4.25j), -4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25+0j, 0j), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25j, 0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25+0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25j), -4.25, 0.0) check(complex(real=4.25), 4.25, 0.0) - check(complex(real=4.25+0j), 4.25, 0.0) - check(complex(real=4.25+1.5j), 4.25, 1.5) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(real=4.25+0j), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(real=4.25+1.5j), 4.25, 1.5) check(complex(imag=1.5), 0.0, 1.5) check(complex(real=4.25, imag=1.5), 4.25, 1.5) check(complex(4.25, imag=1.5), 4.25, 1.5) @@ -420,22 +448,22 @@ def check(z, x, y): del c, c2 self.assertRaisesRegex(TypeError, - "first argument must be a string or a number, not 'dict'", + "argument must be a string or a number, not dict", complex, {}) self.assertRaisesRegex(TypeError, - "first argument must be a string or a number, not 'NoneType'", + "argument must be a string or a number, not NoneType", complex, None) self.assertRaisesRegex(TypeError, - "first argument must be a string or a number, not 'dict'", + "argument 'real' must be a real number, not dict", complex, {1:2}, 0) self.assertRaisesRegex(TypeError, - "can't take second arg if first is a string", + "argument 'real' must be a real number, not str", complex, '1', 0) self.assertRaisesRegex(TypeError, - "second argument must be a number, not 'dict'", + "argument 'imag' must be a real number, not dict", complex, 0, {1:2}) self.assertRaisesRegex(TypeError, - "second arg can't be a string", + "argument 'imag' must be a real number, not str", complex, 0, '1') self.assertRaises(TypeError, complex, WithComplex(1.5)) diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 3648a8982a37e0..28607ee37000f9 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -806,7 +806,10 @@ def testMixedMultiplication(self): self.assertTypedEquals(F(3, 2) * Polar(4, 2), Polar(F(6, 1), 2)) self.assertTypedEquals(F(3, 2) * Polar(4.0, 2), Polar(6.0, 2)) self.assertTypedEquals(F(3, 2) * Rect(4, 3), Rect(F(6, 1), F(9, 2))) - self.assertTypedEquals(F(3, 2) * RectComplex(4, 3), RectComplex(6.0+0j, 4.5+0j)) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + self.assertTypedEquals(F(3, 2) * RectComplex(4, 3), + RectComplex(6.0+0j, 4.5+0j)) self.assertRaises(TypeError, operator.mul, Polar(4, 2), F(3, 2)) self.assertTypedEquals(Rect(4, 3) * F(3, 2), 6.0 + 4.5j) self.assertEqual(F(3, 2) * SymbolicComplex('X'), SymbolicComplex('3/2 * X')) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-27-19-13-49.gh-issue-109218.-sdDg0.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-27-19-13-49.gh-issue-109218.-sdDg0.rst new file mode 100644 index 00000000000000..db762174a8c1e1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-27-19-13-49.gh-issue-109218.-sdDg0.rst @@ -0,0 +1,3 @@ +:func:`complex` accepts now a string only as a positional argument. Passing +a complex number as the "real" or "imag" argument is deprecated; it should +only be passed as a single positional argument. diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 17ee43725dd70d..59c84f1359b966 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -894,8 +894,8 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) } else { PyErr_Format(PyExc_TypeError, - "complex() argument must be a string or a number, not '%.200s'", - Py_TYPE(v)->tp_name); + "complex() argument must be a string or a number, not %T", + v); return NULL; } @@ -905,6 +905,77 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) return result; } +/* The constructor should only accept a string as a positional argument, + * not as by the 'real' keyword. But Argument Clinic does not allow + * to distinguish between argument passed positionally and by keyword. + * So the constructor must be split into two parts: actual_complex_new() + * handles the case of no arguments and one positional argument, and calls + * complex_new(), implemented with Argument Clinic, to handle the remaining + * cases: 'real' and 'imag' arguments. This separation is well suited + * for different constructor roles: convering a string or number to a complex + * number and constructing a complex number from real and imaginary parts. + */ +static PyObject * +actual_complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *res = NULL; + PyNumberMethods *nbr; + + if (PyTuple_GET_SIZE(args) > 1 || (kwargs != NULL && PyDict_GET_SIZE(kwargs))) { + return complex_new(type, args, kwargs); + } + if (!PyTuple_GET_SIZE(args)) { + return complex_subtype_from_doubles(type, 0, 0); + } + + PyObject *arg = PyTuple_GET_ITEM(args, 0); + /* Special-case for a single argument when type(arg) is complex. */ + if (PyComplex_CheckExact(arg) && type == &PyComplex_Type) { + /* Note that we can't know whether it's safe to return + a complex *subclass* instance as-is, hence the restriction + to exact complexes here. If either the input or the + output is a complex subclass, it will be handled below + as a non-orthogonal vector. */ + return Py_NewRef(arg); + } + if (PyUnicode_Check(arg)) { + return complex_subtype_from_string(type, arg); + } + PyObject *tmp = try_complex_special_method(arg); + if (tmp) { + Py_complex c = ((PyComplexObject*)tmp)->cval; + res = complex_subtype_from_doubles(type, c.real, c.imag); + Py_DECREF(tmp); + } + else if (PyErr_Occurred()) { + return NULL; + } + else if (PyComplex_Check(arg)) { + /* Note that if arg is of a complex subtype, we're only + retaining its real & imag parts here, and the return + value is (properly) of the builtin complex type. */ + Py_complex c = ((PyComplexObject*)arg)->cval; + res = complex_subtype_from_doubles(type, c.real, c.imag); + } + else if ((nbr = Py_TYPE(arg)->tp_as_number) != NULL && + (nbr->nb_float != NULL || nbr->nb_index != NULL)) + { + /* The argument really is entirely real, and contributes + nothing in the imaginary direction. + Just treat it as a double. */ + double r = PyFloat_AsDouble(arg); + if (r != -1.0 || !PyErr_Occurred()) { + res = complex_subtype_from_doubles(type, r, 0); + } + } + else { + PyErr_Format(PyExc_TypeError, + "complex() argument must be a string or a number, not %T", + arg); + } + return res; +} + /*[clinic input] @classmethod complex.__new__ as complex_new @@ -933,32 +1004,10 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) if (r == NULL) { r = _PyLong_GetZero(); } + PyObject *orig_r = r; - /* Special-case for a single argument when type(arg) is complex. */ - if (PyComplex_CheckExact(r) && i == NULL && - type == &PyComplex_Type) { - /* Note that we can't know whether it's safe to return - a complex *subclass* instance as-is, hence the restriction - to exact complexes here. If either the input or the - output is a complex subclass, it will be handled below - as a non-orthogonal vector. */ - return Py_NewRef(r); - } - if (PyUnicode_Check(r)) { - if (i != NULL) { - PyErr_SetString(PyExc_TypeError, - "complex() can't take second arg" - " if first is a string"); - return NULL; - } - return complex_subtype_from_string(type, r); - } - if (i != NULL && PyUnicode_Check(i)) { - PyErr_SetString(PyExc_TypeError, - "complex() second arg can't be a string"); - return NULL; - } - + /* DEPRECATED: The call of try_complex_special_method() for the "real" + * part will be dropped after the end of the deprecation period. */ tmp = try_complex_special_method(r); if (tmp) { r = tmp; @@ -973,9 +1022,8 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) (nbr->nb_float == NULL && nbr->nb_index == NULL && !PyComplex_Check(r))) { PyErr_Format(PyExc_TypeError, - "complex() first argument must be a string or a number, " - "not '%.200s'", - Py_TYPE(r)->tp_name); + "complex() argument 'real' must be a real number, not %T", + r); if (own_r) { Py_DECREF(r); } @@ -987,9 +1035,8 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) (nbi->nb_float == NULL && nbi->nb_index == NULL && !PyComplex_Check(i))) { PyErr_Format(PyExc_TypeError, - "complex() second argument must be a number, " - "not '%.200s'", - Py_TYPE(i)->tp_name); + "complex() argument 'imag' must be a real number, not %T", + i); if (own_r) { Py_DECREF(r); } @@ -1001,6 +1048,7 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) both be treated as numbers, and the constructor should return a complex number equal to (real + imag*1j). + The following is DEPRECATED: Note that we do NOT assume the input to already be in canonical form; the "real" and "imag" parts might themselves be complex numbers, which slightly complicates the code below. */ @@ -1011,19 +1059,27 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) cr = ((PyComplexObject*)r)->cval; cr_is_complex = 1; if (own_r) { + /* r was a newly created complex number, rather + than the original "real" argument. */ Py_DECREF(r); } + nbr = Py_TYPE(orig_r)->tp_as_number; + if (nbr == NULL || + (nbr->nb_float == NULL && nbr->nb_index == NULL)) + { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "complex() argument 'real' must be a real number, not %T", + orig_r)) { + return NULL; + } + } } else { /* The "real" part really is entirely real, and contributes nothing in the imaginary direction. Just treat it as a double. */ tmp = PyNumber_Float(r); - if (own_r) { - /* r was a newly created complex number, rather - than the original "real" argument. */ - Py_DECREF(r); - } + assert(!own_r); if (tmp == NULL) return NULL; assert(PyFloat_Check(tmp)); @@ -1035,6 +1091,11 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) ci.real = cr.imag; } else if (PyComplex_Check(i)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "complex() argument 'imag' must be a real number, not %T", + i)) { + return NULL; + } ci = ((PyComplexObject*)i)->cval; ci_is_complex = 1; } else { @@ -1134,6 +1195,6 @@ PyTypeObject PyComplex_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - complex_new, /* tp_new */ + actual_complex_new, /* tp_new */ PyObject_Del, /* tp_free */ }; From 9732ed5ca94cd8fe9ca2fc7ba5a42dfa2b7791ea Mon Sep 17 00:00:00 2001 From: James De Bias <81095953+DBJim@users.noreply.github.com> Date: Fri, 31 May 2024 06:34:59 +1000 Subject: [PATCH 287/903] gh-107262: Update Tkinter tests for Tcl/Tk 8.6.14 (GH-119322) Co-authored-by: Serhiy Storchaka --- Lib/test/test_tkinter/test_widgets.py | 14 ++++++++++---- Lib/test/test_ttk/test_widgets.py | 17 ++++++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index 85bf5ff7652b69..f5f2fd2ee37b84 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -660,7 +660,9 @@ def test_configure_tabs(self): widget = self.create() self.checkParam(widget, 'tabs', (10.2, 20.7, '1i', '2i')) self.checkParam(widget, 'tabs', '10.2 20.7 1i 2i', - expected=('10.2', '20.7', '1i', '2i')) + expected=(10.2, 20.7, '1i', '2i') + if get_tk_patchlevel(self.root) >= (8, 6, 14) + else ('10.2', '20.7', '1i', '2i')) self.checkParam(widget, 'tabs', '2c left 4c 6c center', expected=('2c', 'left', '4c', '6c', 'center')) self.checkInvalidParam(widget, 'tabs', 'spam', @@ -999,12 +1001,16 @@ def test_itemconfigure(self): widget.itemconfigure() with self.assertRaisesRegex(TclError, 'bad listbox index "red"'): widget.itemconfigure('red') + if get_tk_patchlevel(self.root) >= (8, 6, 14): + prefix = ('background', '', '', '') + else: + prefix = ('background', 'background', 'Background', '') self.assertEqual(widget.itemconfigure(0, 'background'), - ('background', 'background', 'Background', '', 'red')) + (*prefix, 'red')) self.assertEqual(widget.itemconfigure('end', 'background'), - ('background', 'background', 'Background', '', 'violet')) + (*prefix, 'violet')) self.assertEqual(widget.itemconfigure('@0,0', 'background'), - ('background', 'background', 'Background', '', 'red')) + (*prefix, 'red')) d = widget.itemconfigure(0) self.assertIsInstance(d, dict) diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index ca7402b276013d..2e0702da5448a8 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -27,13 +27,20 @@ def test_configure_class(self): def test_configure_padding(self): widget = self.create() - self.checkParam(widget, 'padding', 0, expected=('0',)) - self.checkParam(widget, 'padding', 5, expected=('5',)) - self.checkParam(widget, 'padding', (5, 6), expected=('5', '6')) + if get_tk_patchlevel(self.root) < (8, 6, 14): + def padding_conv(value): + self.assertIsInstance(value, tuple) + return tuple(map(str, value)) + else: + padding_conv = None + self.checkParam(widget, 'padding', 0, expected=(0,), conv=padding_conv) + self.checkParam(widget, 'padding', 5, expected=(5,), conv=padding_conv) + self.checkParam(widget, 'padding', (5, 6), + expected=(5, 6), conv=padding_conv) self.checkParam(widget, 'padding', (5, 6, 7), - expected=('5', '6', '7')) + expected=(5, 6, 7), conv=padding_conv) self.checkParam(widget, 'padding', (5, 6, 7, 8), - expected=('5', '6', '7', '8')) + expected=(5, 6, 7, 8), conv=padding_conv) self.checkParam(widget, 'padding', ('5p', '6p', '7p', '8p')) self.checkParam(widget, 'padding', (), expected='') From 13a5fdc72f701c053b96abea48cd8f2775e9418e Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 30 May 2024 21:55:06 +0100 Subject: [PATCH 288/903] gh-119744: move a few functions from compile.c to flowgraph.c (#119745) --- Include/internal/pycore_compile.h | 6 -- Include/internal/pycore_flowgraph.h | 7 ++ Modules/_testinternalcapi.c | 1 + Python/compile.c | 155 ++-------------------------- Python/flowgraph.c | 145 +++++++++++++++++++++++++- 5 files changed, 157 insertions(+), 157 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 3c21f83a18b52a..a1ac034e3e44af 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -101,12 +101,6 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( int optimize, int compile_mode); -// Export for '_testinternalcapi' shared extension -PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( - PyObject *instructions, - PyObject *consts, - int nlocals); - // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyCodeObject*) _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 819117b83114bc..5043260d2fd99f 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -24,6 +24,7 @@ int _PyCfgBuilder_CheckSize(struct _PyCfgBuilder* g); int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, int nlocals, int nparams, int firstlineno); +struct _PyCfgBuilder* _PyCfg_FromInstructionSequence(_PyInstructionSequence *seq); int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyInstructionSequence *seq); int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, int code_flags, int *stackdepth, int *nlocalsplus, @@ -34,6 +35,12 @@ _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache PyObject *consts, int maxdepth, _PyInstructionSequence *instrs, int nlocalsplus, int code_flags, PyObject *filename); +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( + PyObject *instructions, + PyObject *consts, + int nlocals); + #ifdef __cplusplus } #endif diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 129c136906739d..d9b9c999603d5a 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -18,6 +18,7 @@ #include "pycore_context.h" // _PyContext_NewHamtForTests() #include "pycore_dict.h" // _PyManagedDictPointer_GetValues() #include "pycore_fileutils.h" // _Py_normpath() +#include "pycore_flowgraph.h" // _PyCompile_OptimizeCfg() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() diff --git a/Python/compile.c b/Python/compile.c index 3a80577e0f2b2d..7d74096fcdf94e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -197,47 +197,6 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, return SUCCESS; } -static cfg_builder* -instr_sequence_to_cfg(instr_sequence *seq) { - if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { - return NULL; - } - cfg_builder *g = _PyCfgBuilder_New(); - if (g == NULL) { - return NULL; - } - for (int i = 0; i < seq->s_used; i++) { - seq->s_instrs[i].i_target = 0; - } - for (int i = 0; i < seq->s_used; i++) { - instruction *instr = &seq->s_instrs[i]; - if (HAS_TARGET(instr->i_opcode)) { - assert(instr->i_oparg >= 0 && instr->i_oparg < seq->s_used); - seq->s_instrs[instr->i_oparg].i_target = 1; - } - } - for (int i = 0; i < seq->s_used; i++) { - instruction *instr = &seq->s_instrs[i]; - if (instr->i_target) { - jump_target_label lbl_ = {i}; - if (_PyCfgBuilder_UseLabel(g, lbl_) < 0) { - goto error; - } - } - int opcode = instr->i_opcode; - int oparg = instr->i_oparg; - if (_PyCfgBuilder_Addop(g, opcode, oparg, instr->i_loc) < 0) { - goto error; - } - } - if (_PyCfgBuilder_CheckSize(g) < 0) { - goto error; - } - return g; -error: - _PyCfgBuilder_Free(g); - return NULL; -} /* The following items change on entry and exit of code blocks. They must be saved and restored when returning to a block. @@ -691,48 +650,6 @@ compiler_set_qualname(struct compiler *c) return SUCCESS; } -/* Return the stack effect of opcode with argument oparg. - - Some opcodes have different stack effect when jump to the target and - when not jump. The 'jump' parameter specifies the case: - - * 0 -- when not jump - * 1 -- when jump - * -1 -- maximal - */ -static int -stack_effect(int opcode, int oparg, int jump) -{ - if (opcode < 0) { - return PY_INVALID_STACK_EFFECT; - } - if ((opcode <= MAX_REAL_OPCODE) && (_PyOpcode_Deopt[opcode] != opcode)) { - // Specialized instructions are not supported. - return PY_INVALID_STACK_EFFECT; - } - int popped = _PyOpcode_num_popped(opcode, oparg); - int pushed = _PyOpcode_num_pushed(opcode, oparg); - if (popped < 0 || pushed < 0) { - return PY_INVALID_STACK_EFFECT; - } - if (IS_BLOCK_PUSH_OPCODE(opcode) && !jump) { - return 0; - } - return pushed - popped; -} - -int -PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) -{ - return stack_effect(opcode, oparg, jump); -} - -int -PyCompile_OpcodeStackEffect(int opcode, int oparg) -{ - return stack_effect(opcode, oparg, -1); -} - int _PyCompile_OpcodeIsValid(int opcode) { @@ -7592,7 +7509,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, if (consts == NULL) { goto error; } - g = instr_sequence_to_cfg(u->u_instr_sequence); + g = _PyCfg_FromInstructionSequence(u->u_instr_sequence); if (g == NULL) { goto error; } @@ -7645,39 +7562,6 @@ optimize_and_assemble(struct compiler *c, int addNone) return optimize_and_assemble_code_unit(u, const_cache, code_flags, filename); } -/* Access to compiler optimizations for unit tests. - * - * _PyCompile_CodeGen takes and AST, applies code-gen and - * returns the unoptimized CFG as an instruction list. - * - * _PyCompile_OptimizeCfg takes an instruction list, constructs - * a CFG, optimizes it and converts back to an instruction list. - * - * An instruction list is a PyList where each item is either - * a tuple describing a single instruction: - * (opcode, oparg, lineno, end_lineno, col, end_col), or - * a jump target label marking the beginning of a basic block. - */ - - -static PyObject * -cfg_to_instruction_sequence(cfg_builder *g) -{ - instr_sequence *seq = (instr_sequence *)_PyInstructionSequence_New(); - if (seq != NULL) { - if (_PyCfg_ToInstructionSequence(g, seq) < 0) { - goto error; - } - if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { - goto error; - } - } - return (PyObject*)seq; -error: - PyInstructionSequence_Fini(seq); - return NULL; -} - // C implementation of inspect.cleandoc() // // Difference from inspect.cleandoc(): @@ -7768,6 +7652,12 @@ _PyCompile_CleanDoc(PyObject *doc) return res; } +/* Access to compiler optimizations for unit tests. + * + * _PyCompile_CodeGen takes an AST, applies code-gen and + * returns the unoptimized CFG as an instruction list. + * + */ PyObject * _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, @@ -7859,35 +7749,6 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return res; } -PyObject * -_PyCompile_OptimizeCfg(PyObject *seq, PyObject *consts, int nlocals) -{ - if (!_PyInstructionSequence_Check(seq)) { - PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); - return NULL; - } - PyObject *const_cache = PyDict_New(); - if (const_cache == NULL) { - return NULL; - } - - PyObject *res = NULL; - cfg_builder *g = instr_sequence_to_cfg((instr_sequence*)seq); - if (g == NULL) { - goto error; - } - int nparams = 0, firstlineno = 1; - if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, - nparams, firstlineno) < 0) { - goto error; - } - res = cfg_to_instruction_sequence(g); -error: - Py_DECREF(const_cache); - _PyCfgBuilder_Free(g); - return res; -} - int _PyCfg_JumpLabelsToTargets(cfg_builder *g); PyCodeObject * @@ -7908,7 +7769,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, return NULL; } - g = instr_sequence_to_cfg((instr_sequence*)seq); + g = _PyCfg_FromInstructionSequence((instr_sequence*)seq); if (g == NULL) { goto error; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 83768023a4d870..b0c8004130fb07 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -751,6 +751,36 @@ make_cfg_traversal_stack(basicblock *entryblock) { return stack; } +/* Return the stack effect of opcode with argument oparg. + + Some opcodes have different stack effect when jump to the target and + when not jump. The 'jump' parameter specifies the case: + + * 0 -- when not jump + * 1 -- when jump + * -1 -- maximal + */ +Py_LOCAL(int) +stack_effect(int opcode, int oparg, int jump) +{ + if (opcode < 0) { + return PY_INVALID_STACK_EFFECT; + } + if ((opcode <= MAX_REAL_OPCODE) && (_PyOpcode_Deopt[opcode] != opcode)) { + // Specialized instructions are not supported. + return PY_INVALID_STACK_EFFECT; + } + int popped = _PyOpcode_num_popped(opcode, oparg); + int pushed = _PyOpcode_num_pushed(opcode, oparg); + if (popped < 0 || pushed < 0) { + return PY_INVALID_STACK_EFFECT; + } + if (IS_BLOCK_PUSH_OPCODE(opcode) && !jump) { + return 0; + } + return pushed - popped; +} + Py_LOCAL_INLINE(int) stackdepth_push(basicblock ***sp, basicblock *b, int depth) { @@ -795,8 +825,7 @@ calculate_stackdepth(cfg_builder *g) basicblock *next = b->b_next; for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - int effect = PyCompile_OpcodeStackEffectWithJump( - instr->i_opcode, instr->i_oparg, 0); + int effect = stack_effect(instr->i_opcode, instr->i_oparg, 0); if (effect == PY_INVALID_STACK_EFFECT) { PyErr_Format(PyExc_SystemError, "Invalid stack effect for opcode=%d, arg=%i", @@ -813,8 +842,7 @@ calculate_stackdepth(cfg_builder *g) maxdepth = new_depth; } if (HAS_TARGET(instr->i_opcode)) { - effect = PyCompile_OpcodeStackEffectWithJump( - instr->i_opcode, instr->i_oparg, 1); + effect = stack_effect(instr->i_opcode, instr->i_oparg, 1); if (effect == PY_INVALID_STACK_EFFECT) { PyErr_Format(PyExc_SystemError, "Invalid stack effect for opcode=%d, arg=%i", @@ -2711,6 +2739,49 @@ prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_fl return nlocalsplus; } +cfg_builder * +_PyCfg_FromInstructionSequence(_PyInstructionSequence *seq) +{ + if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { + return NULL; + } + cfg_builder *g = _PyCfgBuilder_New(); + if (g == NULL) { + return NULL; + } + for (int i = 0; i < seq->s_used; i++) { + seq->s_instrs[i].i_target = 0; + } + for (int i = 0; i < seq->s_used; i++) { + _PyInstruction *instr = &seq->s_instrs[i]; + if (HAS_TARGET(instr->i_opcode)) { + assert(instr->i_oparg >= 0 && instr->i_oparg < seq->s_used); + seq->s_instrs[instr->i_oparg].i_target = 1; + } + } + for (int i = 0; i < seq->s_used; i++) { + _PyInstruction *instr = &seq->s_instrs[i]; + if (instr->i_target) { + jump_target_label lbl_ = {i}; + if (_PyCfgBuilder_UseLabel(g, lbl_) < 0) { + goto error; + } + } + int opcode = instr->i_opcode; + int oparg = instr->i_oparg; + if (_PyCfgBuilder_Addop(g, opcode, oparg, instr->i_loc) < 0) { + goto error; + } + } + if (_PyCfgBuilder_CheckSize(g) < 0) { + goto error; + } + return g; +error: + _PyCfgBuilder_Free(g); + return NULL; +} + int _PyCfg_ToInstructionSequence(cfg_builder *g, _PyInstructionSequence *seq) { @@ -2742,6 +2813,9 @@ _PyCfg_ToInstructionSequence(cfg_builder *g, _PyInstructionSequence *seq) } } } + if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { + return ERROR; + } return SUCCESS; } @@ -2796,3 +2870,66 @@ _PyCfg_JumpLabelsToTargets(cfg_builder *g) RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); return SUCCESS; } + +/* Exported API functions */ + +int +PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) +{ + return stack_effect(opcode, oparg, jump); +} + +int +PyCompile_OpcodeStackEffect(int opcode, int oparg) +{ + return stack_effect(opcode, oparg, -1); +} + +/* Access to compiler optimizations for unit tests. + + * _PyCompile_OptimizeCfg takes an instruction list, constructs + * a CFG, optimizes it and converts back to an instruction list. + */ + +static PyObject * +cfg_to_instruction_sequence(cfg_builder *g) +{ + _PyInstructionSequence *seq = (_PyInstructionSequence *)_PyInstructionSequence_New(); + if (seq == NULL) { + return NULL; + } + if (_PyCfg_ToInstructionSequence(g, seq) < 0) { + PyInstructionSequence_Fini(seq); + return NULL; + } + return (PyObject*)seq; +} + +PyObject * +_PyCompile_OptimizeCfg(PyObject *seq, PyObject *consts, int nlocals) +{ + if (!_PyInstructionSequence_Check(seq)) { + PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); + return NULL; + } + PyObject *const_cache = PyDict_New(); + if (const_cache == NULL) { + return NULL; + } + + PyObject *res = NULL; + cfg_builder *g = _PyCfg_FromInstructionSequence((_PyInstructionSequence*)seq); + if (g == NULL) { + goto error; + } + int nparams = 0, firstlineno = 1; + if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, + nparams, firstlineno) < 0) { + goto error; + } + res = cfg_to_instruction_sequence(g); +error: + Py_DECREF(const_cache); + _PyCfgBuilder_Free(g); + return res; +} From 0d07182821fad7b95a043d006f1ce13a2d22edcb Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 31 May 2024 00:49:03 -0700 Subject: [PATCH 289/903] gh-111201: Support pyrepl on Windows (#119559) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Anthony Shaw Co-authored-by: Łukasz Langa --- Doc/whatsnew/3.13.rst | 13 +- Lib/_pyrepl/__main__.py | 6 +- Lib/_pyrepl/console.py | 30 +- Lib/_pyrepl/reader.py | 16 +- Lib/_pyrepl/readline.py | 11 +- Lib/_pyrepl/simple_interact.py | 6 +- Lib/_pyrepl/unix_console.py | 28 +- Lib/_pyrepl/windows_console.py | 587 ++++++++++++++++++ Lib/test/test_pyrepl/__init__.py | 12 +- Lib/test/test_pyrepl/support.py | 2 +- Lib/test/test_pyrepl/test_pyrepl.py | 5 +- Lib/test/test_pyrepl/test_unix_console.py | 11 +- Lib/test/test_pyrepl/test_unix_eventqueue.py | 10 +- Lib/test/test_pyrepl/test_windows_console.py | 331 ++++++++++ ...-05-25-18-43-10.gh-issue-111201.SLPJIx.rst | 1 + 15 files changed, 1020 insertions(+), 49 deletions(-) create mode 100644 Lib/_pyrepl/windows_console.py create mode 100644 Lib/test/test_pyrepl/test_windows_console.py create mode 100644 Misc/NEWS.d/next/Windows/2024-05-25-18-43-10.gh-issue-111201.SLPJIx.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 241c07e781af1f..29bb3b81f6323c 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -154,10 +154,10 @@ New Features A Better Interactive Interpreter -------------------------------- -On Unix-like systems like Linux or macOS, Python now uses a new -:term:`interactive` shell. When the user starts the :term:`REPL` from an -interactive terminal, and both :mod:`curses` and :mod:`readline` are -available, the interactive shell now supports the following new features: +On Unix-like systems like Linux or macOS as well as Windows, Python now +uses a new :term:`interactive` shell. When the user starts the +:term:`REPL` from an interactive terminal the interactive shell now +supports the following new features: * Colorized prompts. * Multiline editing with history preservation. @@ -174,10 +174,13 @@ available, the interactive shell now supports the following new features: If the new interactive shell is not desired, it can be disabled via the :envvar:`PYTHON_BASIC_REPL` environment variable. +The new shell requires :mod:`curses` on Unix-like systems. + For more on interactive mode, see :ref:`tut-interac`. (Contributed by Pablo Galindo Salgado, Łukasz Langa, and -Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project.) +Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project. +Windows support contributed by Dino Viehland and Anthony Shaw.) .. _whatsnew313-improved-error-messages: diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index c598019e7cd4ad..dae4ba6e178b9a 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -1,7 +1,11 @@ import os import sys -CAN_USE_PYREPL = sys.platform != "win32" +CAN_USE_PYREPL: bool +if sys.platform != "win32": + CAN_USE_PYREPL = True +else: + CAN_USE_PYREPL = sys.getwindowsversion().build >= 10586 # Windows 10 TH2 def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index d7e86e768671dc..fcabf785069ecb 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -19,10 +19,18 @@ from __future__ import annotations +import sys + from abc import ABC, abstractmethod from dataclasses import dataclass, field +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + + @dataclass class Event: evt: str @@ -36,6 +44,25 @@ class Console(ABC): height: int = 25 width: int = 80 + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + self.encoding = encoding or sys.getdefaultencoding() + + if isinstance(f_in, int): + self.input_fd = f_in + else: + self.input_fd = f_in.fileno() + + if isinstance(f_out, int): + self.output_fd = f_out + else: + self.output_fd = f_out.fileno() + @abstractmethod def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... @@ -108,5 +135,4 @@ def wait(self) -> None: ... @abstractmethod - def repaint(self) -> None: - ... + def repaint(self) -> None: ... diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index d2960bbb6121b3..0045425cdddb79 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -442,14 +442,13 @@ def get_arg(self, default: int = 1) -> int: """ if self.arg is None: return default - else: - return self.arg + return self.arg def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: """Return what should be in the left-hand margin for line 'lineno'.""" if self.arg is not None and cursor_on_line: - prompt = "(arg: %s) " % self.arg + prompt = f"(arg: {self.arg}) " elif self.paste_mode: prompt = "(paste) " elif "\n" in self.buffer: @@ -515,12 +514,12 @@ def pos2xy(self) -> tuple[int, int]: offset = l - 1 if in_wrapped_line else l # need to remove backslash if offset >= pos: break + + if p + sum(l2) >= self.console.width: + pos -= l - 1 # -1 cause backslash is not in buffer else: - if p + sum(l2) >= self.console.width: - pos -= l - 1 # -1 cause backslash is not in buffer - else: - pos -= l + 1 # +1 cause newline is in buffer - y += 1 + pos -= l + 1 # +1 cause newline is in buffer + y += 1 return p + sum(l2[:pos]), y def insert(self, text: str | list[str]) -> None: @@ -582,7 +581,6 @@ def suspend(self) -> SimpleContextManager: for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"): setattr(self, arg, prev_state[arg]) self.prepare() - pass def finish(self) -> None: """Called when a command signals that we're finished.""" diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index ffa14a9ce31a8f..248f3854a29689 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -38,7 +38,14 @@ from . import commands, historical_reader from .completing_reader import CompletingReader -from .unix_console import UnixConsole, _error +from .console import Console as ConsoleType + +Console: type[ConsoleType] +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import UnixConsole as Console, _error +except ImportError: + from .windows_console import WindowsConsole as Console, _error ENCODING = sys.getdefaultencoding() or "latin1" @@ -328,7 +335,7 @@ def __post_init__(self) -> None: def get_reader(self) -> ReadlineAlikeReader: if self.reader is None: - console = UnixConsole(self.f_in, self.f_out, encoding=ENCODING) + console = Console(self.f_in, self.f_out, encoding=ENCODING) self.reader = ReadlineAlikeReader(console=console, config=self.config) return self.reader diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 11e831c1d6c5d4..c624f6e12a7094 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -34,8 +34,12 @@ from types import ModuleType from .readline import _get_reader, multiline_input -from .unix_console import _error +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import _error +except ModuleNotFoundError: + from .windows_console import _error def check() -> str: """Returns the error message if there is a problem initializing the state.""" diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index ec7d0636b9aeb3..4bdb02261982c3 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -143,18 +143,7 @@ def __init__( - term (str): Terminal name. - encoding (str): Encoding to use for I/O operations. """ - - self.encoding = encoding or sys.getdefaultencoding() - - if isinstance(f_in, int): - self.input_fd = f_in - else: - self.input_fd = f_in.fileno() - - if isinstance(f_out, int): - self.output_fd = f_out - else: - self.output_fd = f_out.fileno() + super().__init__(f_in, f_out, term, encoding) self.pollob = poll() self.pollob.register(self.input_fd, select.POLLIN) @@ -592,14 +581,19 @@ def __write_changed_line(self, y, oldline, newline, px_coord): px_pos = 0 j = 0 for c in oldline: - if j >= px_coord: break + if j >= px_coord: + break j += wlen(c) px_pos += 1 # reuse the oldline as much as possible, but stop as soon as we # encounter an ESCAPE, because it might be the start of an escape # sequene - while x_coord < minlen and oldline[x_pos] == newline[x_pos] and newline[x_pos] != "\x1b": + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): x_coord += wlen(newline[x_pos]) x_pos += 1 @@ -619,7 +613,11 @@ def __write_changed_line(self, y, oldline, newline, px_coord): self.__posxy = x_coord + character_width, y # if it's a single character change in the middle of the line - elif x_coord < minlen and oldline[x_pos + 1 :] == newline[x_pos + 1 :] and wlen(oldline[x_pos]) == wlen(newline[x_pos]): + elif ( + x_coord < minlen + and oldline[x_pos + 1 :] == newline[x_pos + 1 :] + and wlen(oldline[x_pos]) == wlen(newline[x_pos]) + ): character_width = wlen(newline[x_pos]) self.__move(x_coord, y) self.__write(newline[x_pos]) diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py new file mode 100644 index 00000000000000..2277865e3262fc --- /dev/null +++ b/Lib/_pyrepl/windows_console.py @@ -0,0 +1,587 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import io +from multiprocessing import Value +import os +import sys + +from abc import ABC, abstractmethod +from collections import deque +from dataclasses import dataclass, field +import ctypes +from ctypes.wintypes import ( + _COORD, + WORD, + SMALL_RECT, + BOOL, + HANDLE, + CHAR, + DWORD, + WCHAR, + SHORT, +) +from ctypes import Structure, POINTER, Union +from .console import Event, Console +from .trace import trace +from .utils import wlen + +try: + from ctypes import GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] +except: + # Keep MyPy happy off Windows + from ctypes import CDLL as WinDLL, cdll as windll + + def GetLastError() -> int: + return 42 + + class WinError(OSError): # type: ignore[no-redef] + def __init__(self, err: int | None, descr: str | None = None) -> None: + self.err = err + self.descr = descr + + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + +VK_MAP: dict[int, str] = { + 0x23: "end", # VK_END + 0x24: "home", # VK_HOME + 0x25: "left", # VK_LEFT + 0x26: "up", # VK_UP + 0x27: "right", # VK_RIGHT + 0x28: "down", # VK_DOWN + 0x2E: "delete", # VK_DELETE + 0x70: "f1", # VK_F1 + 0x71: "f2", # VK_F2 + 0x72: "f3", # VK_F3 + 0x73: "f4", # VK_F4 + 0x74: "f5", # VK_F5 + 0x75: "f6", # VK_F6 + 0x76: "f7", # VK_F7 + 0x77: "f8", # VK_F8 + 0x78: "f9", # VK_F9 + 0x79: "f10", # VK_F10 + 0x7A: "f11", # VK_F11 + 0x7B: "f12", # VK_F12 + 0x7C: "f13", # VK_F13 + 0x7D: "f14", # VK_F14 + 0x7E: "f15", # VK_F15 + 0x7F: "f16", # VK_F16 + 0x79: "f17", # VK_F17 + 0x80: "f18", # VK_F18 + 0x81: "f19", # VK_F19 + 0x82: "f20", # VK_F20 +} + +# Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences +ERASE_IN_LINE = "\x1b[K" +MOVE_LEFT = "\x1b[{}D" +MOVE_RIGHT = "\x1b[{}C" +MOVE_UP = "\x1b[{}A" +MOVE_DOWN = "\x1b[{}B" +CLEAR = "\x1b[H\x1b[J" + + +class _error(Exception): + pass + + +class WindowsConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + super().__init__(f_in, f_out, term, encoding) + + SetConsoleMode( + OutHandle, + ENABLE_WRAP_AT_EOL_OUTPUT + | ENABLE_PROCESSED_OUTPUT + | ENABLE_VIRTUAL_TERMINAL_PROCESSING, + ) + self.screen: list[str] = [] + self.width = 80 + self.height = 25 + self.__offset = 0 + self.event_queue: deque[Event] = deque() + try: + self.out = io._WindowsConsoleIO(self.output_fd, "w") # type: ignore[attr-defined] + except ValueError: + # Console I/O is redirected, fallback... + self.out = None + + def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + + while len(self.screen) < min(len(screen), self.height): + self._hide_cursor() + self._move_relative(0, len(self.screen) - 1) + self.__write("\n") + self.__posxy = 0, len(self.screen) + self.screen.append("") + + px, py = self.__posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + scroll_lines = offset - old_offset + + # Scrolling the buffer as the current input is greater than the visible + # portion of the window. We need to scroll the visible portion and the + # entire history + self._scroll(scroll_lines, self._getscrollbacksize()) + self.__posxy = self.__posxy[0], self.__posxy[1] + scroll_lines + self.__offset += scroll_lines + + for i in range(scroll_lines): + self.screen.append("") + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + self.__offset = offset + + self._hide_cursor() + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self._move_relative(0, y) + self.__posxy = 0, y + self._erase_to_end() + y += 1 + + self._show_cursor() + + self.screen = screen + self.move_cursor(cx, cy) + + def __write_changed_line( + self, y: int, oldline: str, newline: str, px_coord: int + ) -> None: + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + self._hide_cursor() + self._move_relative(x_coord, y) + if wlen(oldline) > wlen(newline): + self._erase_to_end() + + self.__write(newline[x_pos:]) + if wlen(newline) == self.width: + # If we wrapped we want to start at the next line + self._move_relative(0, y + 1) + self.__posxy = 0, y + 1 + else: + self.__posxy = wlen(newline), y + + if "\x1b" in newline or y != self.__posxy[1]: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def _scroll( + self, top: int, bottom: int, left: int | None = None, right: int | None = None + ) -> None: + scroll_rect = SMALL_RECT() + scroll_rect.Top = SHORT(top) + scroll_rect.Bottom = SHORT(bottom) + scroll_rect.Left = SHORT(0 if left is None else left) + scroll_rect.Right = SHORT( + self.getheightwidth()[1] - 1 if right is None else right + ) + destination_origin = _COORD() + fill_info = CHAR_INFO() + fill_info.UnicodeChar = " " + + if not ScrollConsoleScreenBuffer( + OutHandle, scroll_rect, None, destination_origin, fill_info + ): + raise WinError(GetLastError()) + + def _hide_cursor(self): + self.__write("\x1b[?25l") + + def _show_cursor(self): + self.__write("\x1b[?25h") + + def _enable_blinking(self): + self.__write("\x1b[?12h") + + def _disable_blinking(self): + self.__write("\x1b[?12l") + + def __write(self, text: str) -> None: + if self.out is not None: + self.out.write(text.encode(self.encoding, "replace")) + self.out.flush() + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + + @property + def screen_xy(self) -> tuple[int, int]: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return info.dwCursorPosition.X, info.dwCursorPosition.Y + + def _erase_to_end(self) -> None: + self.__write(ERASE_IN_LINE) + + def prepare(self) -> None: + trace("prepare") + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__posxy = 0, 0 + self.__gone_tall = 0 + self.__offset = 0 + + def restore(self) -> None: + pass + + def _move_relative(self, x: int, y: int) -> None: + """Moves relative to the current __posxy""" + dx = x - self.__posxy[0] + dy = y - self.__posxy[1] + if dx < 0: + self.__write(MOVE_LEFT.format(-dx)) + elif dx > 0: + self.__write(MOVE_RIGHT.format(dx)) + + if dy < 0: + self.__write(MOVE_UP.format(-dy)) + elif dy > 0: + self.__write(MOVE_DOWN.format(dy)) + + def move_cursor(self, x: int, y: int) -> None: + if x < 0 or y < 0: + raise ValueError(f"Bad cursor position {x}, {y}") + + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(0, Event("scroll", "")) + else: + self._move_relative(x, y) + self.__posxy = x, y + + def set_cursor_vis(self, visible: bool) -> None: + if visible: + self._show_cursor() + else: + self._hide_cursor() + + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return ( + info.srWindow.Bottom - info.srWindow.Top + 1, + info.srWindow.Right - info.srWindow.Left + 1, + ) + + def _getscrollbacksize(self) -> int: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + + return info.srWindow.Bottom # type: ignore[no-any-return] + + def _read_input(self) -> INPUT_RECORD | None: + rec = INPUT_RECORD() + read = DWORD() + if not ReadConsoleInput(InHandle, rec, 1, read): + raise WinError(GetLastError()) + + if read.value == 0: + return None + + return rec + + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + if self.event_queue: + return self.event_queue.pop() + + while True: + rec = self._read_input() + if rec is None: + if block: + continue + return None + + if rec.EventType == WINDOW_BUFFER_SIZE_EVENT: + return Event("resize", "") + + if rec.EventType != KEY_EVENT or not rec.Event.KeyEvent.bKeyDown: + # Only process keys and keydown events + if block: + continue + return None + + key = rec.Event.KeyEvent.uChar.UnicodeChar + + if rec.Event.KeyEvent.uChar.UnicodeChar == "\r": + # Make enter make unix-like + return Event(evt="key", data="\n", raw=b"\n") + elif rec.Event.KeyEvent.wVirtualKeyCode == 8: + # Turn backspace directly into the command + return Event( + evt="key", + data="backspace", + raw=rec.Event.KeyEvent.uChar.UnicodeChar, + ) + elif rec.Event.KeyEvent.uChar.UnicodeChar == "\x00": + # Handle special keys like arrow keys and translate them into the appropriate command + code = VK_MAP.get(rec.Event.KeyEvent.wVirtualKeyCode) + if code: + return Event( + evt="key", data=code, raw=rec.Event.KeyEvent.uChar.UnicodeChar + ) + if block: + continue + + return None + + return Event(evt="key", data=key, raw=rec.Event.KeyEvent.uChar.UnicodeChar) + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + raise NotImplementedError("push_char not supported on Windows") + + def beep(self) -> None: + self.__write("\x07") + + def clear(self) -> None: + """Wipe the screen""" + self.__write(CLEAR) + self.__posxy = 0, 0 + self.screen = [""] + + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self._move_relative(0, min(y, self.height + self.__offset - 1)) + self.__write("\r\n") + + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere). + + All output on Windows is unbuffered so this is a nop""" + pass + + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + while self._read_input() is not None: + pass + + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + return Event("key", "", b"") + + def wait(self) -> None: + """Wait for an event.""" + raise NotImplementedError("No wait support") + + def repaint(self) -> None: + raise NotImplementedError("No repaint support") + + +# Windows interop +class CONSOLE_SCREEN_BUFFER_INFO(Structure): + _fields_ = [ + ("dwSize", _COORD), + ("dwCursorPosition", _COORD), + ("wAttributes", WORD), + ("srWindow", SMALL_RECT), + ("dwMaximumWindowSize", _COORD), + ] + + +class CONSOLE_CURSOR_INFO(Structure): + _fields_ = [ + ("dwSize", DWORD), + ("bVisible", BOOL), + ] + + +class CHAR_INFO(Structure): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Attributes", WORD), + ] + + +class Char(Union): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Char", CHAR), + ] + + +class KeyEvent(ctypes.Structure): + _fields_ = [ + ("bKeyDown", BOOL), + ("wRepeatCount", WORD), + ("wVirtualKeyCode", WORD), + ("wVirtualScanCode", WORD), + ("uChar", Char), + ("dwControlKeyState", DWORD), + ] + + +class WindowsBufferSizeEvent(ctypes.Structure): + _fields_ = [("dwSize", _COORD)] + + +class ConsoleEvent(ctypes.Union): + _fields_ = [ + ("KeyEvent", KeyEvent), + ("WindowsBufferSizeEvent", WindowsBufferSizeEvent), + ] + + +class INPUT_RECORD(Structure): + _fields_ = [("EventType", WORD), ("Event", ConsoleEvent)] + + +KEY_EVENT = 0x01 +FOCUS_EVENT = 0x10 +MENU_EVENT = 0x08 +MOUSE_EVENT = 0x02 +WINDOW_BUFFER_SIZE_EVENT = 0x04 + +ENABLE_PROCESSED_OUTPUT = 0x01 +ENABLE_WRAP_AT_EOL_OUTPUT = 0x02 +ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 + +STD_INPUT_HANDLE = -10 +STD_OUTPUT_HANDLE = -11 + +if sys.platform == "win32": + _KERNEL32 = WinDLL("kernel32", use_last_error=True) + + GetStdHandle = windll.kernel32.GetStdHandle + GetStdHandle.argtypes = [DWORD] + GetStdHandle.restype = HANDLE + + GetConsoleScreenBufferInfo = _KERNEL32.GetConsoleScreenBufferInfo + GetConsoleScreenBufferInfo.argtypes = [ + HANDLE, + ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + GetConsoleScreenBufferInfo.restype = BOOL + + ScrollConsoleScreenBuffer = _KERNEL32.ScrollConsoleScreenBufferW + ScrollConsoleScreenBuffer.argtypes = [ + HANDLE, + POINTER(SMALL_RECT), + POINTER(SMALL_RECT), + _COORD, + POINTER(CHAR_INFO), + ] + ScrollConsoleScreenBuffer.restype = BOOL + + SetConsoleMode = _KERNEL32.SetConsoleMode + SetConsoleMode.argtypes = [HANDLE, DWORD] + SetConsoleMode.restype = BOOL + + ReadConsoleInput = _KERNEL32.ReadConsoleInputW + ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] + ReadConsoleInput.restype = BOOL + + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) + InHandle = GetStdHandle(STD_INPUT_HANDLE) +else: + + def _win_only(*args, **kwargs): + raise NotImplementedError("Windows only") + + GetStdHandle = _win_only + GetConsoleScreenBufferInfo = _win_only + ScrollConsoleScreenBuffer = _win_only + SetConsoleMode = _win_only + ReadConsoleInput = _win_only + OutHandle = 0 + InHandle = 0 diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py index fa38b86b847dd9..8359d9844623c2 100644 --- a/Lib/test/test_pyrepl/__init__.py +++ b/Lib/test/test_pyrepl/__init__.py @@ -1,12 +1,14 @@ import os +import sys from test.support import requires, load_package_tests from test.support.import_helper import import_module -# Optionally test pyrepl. This currently requires that the -# 'curses' resource be given on the regrtest command line using the -u -# option. Additionally, we need to attempt to import curses and readline. -requires("curses") -curses = import_module("curses") +if sys.platform != "win32": + # On non-Windows platforms, testing pyrepl currently requires that the + # 'curses' resource be given on the regrtest command line using the -u + # option. Additionally, we need to attempt to import curses and readline. + requires("curses") + curses = import_module("curses") def load_tests(*args): diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py index 75539049d43c2a..d2f5429aea7a11 100644 --- a/Lib/test/test_pyrepl/support.py +++ b/Lib/test/test_pyrepl/support.py @@ -55,7 +55,7 @@ def get_prompt(lineno, cursor_on_line) -> str: return reader -def prepare_console(events: Iterable[Event], **kwargs): +def prepare_console(events: Iterable[Event], **kwargs) -> MagicMock | Console: console = MagicMock() console.get_event.side_effect = events console.height = 100 diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index bdcabf9be05b9e..aa2722095794c9 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -508,14 +508,15 @@ def prepare_reader(self, events, namespace): reader = ReadlineAlikeReader(console=console, config=config) return reader + @patch("rlcompleter._readline_available", False) def test_simple_completion(self): - events = code_to_events("os.geten\t\n") + events = code_to_events("os.getpid\t\n") namespace = {"os": os} reader = self.prepare_reader(events, namespace) output = multiline_input(reader, namespace) - self.assertEqual(output, "os.getenv") + self.assertEqual(output, "os.getpid()") def test_completion_with_many_options(self): # Test with something that initially displays many options diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index e1faa00caafc27..d0b98f17ade094 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -1,12 +1,16 @@ import itertools +import sys +import unittest from functools import partial from unittest import TestCase from unittest.mock import MagicMock, call, patch, ANY from .support import handle_all_events, code_to_events -from _pyrepl.console import Event -from _pyrepl.unix_console import UnixConsole - +try: + from _pyrepl.console import Event + from _pyrepl.unix_console import UnixConsole +except ImportError: + pass def unix_console(events, **kwargs): console = UnixConsole() @@ -67,6 +71,7 @@ def unix_console(events, **kwargs): } +@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") @patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) @patch( "_pyrepl.curses.tparm", diff --git a/Lib/test/test_pyrepl/test_unix_eventqueue.py b/Lib/test/test_pyrepl/test_unix_eventqueue.py index c06536b4a86a04..301f79927a741f 100644 --- a/Lib/test/test_pyrepl/test_unix_eventqueue.py +++ b/Lib/test/test_pyrepl/test_unix_eventqueue.py @@ -1,11 +1,15 @@ import tempfile import unittest +import sys from unittest.mock import patch -from _pyrepl.console import Event -from _pyrepl.unix_eventqueue import EventQueue - +try: + from _pyrepl.console import Event + from _pyrepl.unix_eventqueue import EventQueue +except ImportError: + pass +@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") @patch("_pyrepl.curses.tigetstr", lambda x: b"") class TestUnixEventQueue(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py new file mode 100644 index 00000000000000..e87dfe99b1a17d --- /dev/null +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -0,0 +1,331 @@ +import itertools +import sys +import unittest +from _pyrepl.console import Event, Console +from _pyrepl.windows_console import ( + MOVE_LEFT, + MOVE_RIGHT, + MOVE_UP, + MOVE_DOWN, + ERASE_IN_LINE, +) +from functools import partial +from typing import Iterable +from unittest import TestCase, main +from unittest.mock import MagicMock, call, patch, ANY + +from .support import handle_all_events, code_to_events + +try: + from _pyrepl.console import Event + from _pyrepl.windows_console import WindowsConsole +except ImportError: + pass + + +@unittest.skipIf(sys.platform != "win32", "Test class specifically for Windows") +class WindowsConsoleTests(TestCase): + def console(self, events, **kwargs) -> Console: + console = WindowsConsole() + console.get_event = MagicMock(side_effect=events) + console._scroll = MagicMock() + console._hide_cursor = MagicMock() + console._show_cursor = MagicMock() + console._getscrollbacksize = MagicMock(42) + console.out = MagicMock() + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + def handle_events(self, events: Iterable[Event], **kwargs): + return handle_all_events(events, partial(self.console, **kwargs)) + + def handle_events_narrow(self, events): + return self.handle_events(events, width=5) + + def handle_events_short(self, events): + return self.handle_events(events, height=1) + + def handle_events_height_3(self, events): + return self.handle_events(events, height=3) + + def test_simple_addition(self): + code = "12+34" + events = code_to_events(code) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(b"2") + con.out.write.assert_any_call(b"+") + con.out.write.assert_any_call(b"3") + con.out.write.assert_any_call(b"4") + con.restore() + + def test_wrap(self): + code = "12+34" + events = code_to_events(code) + _, con = self.handle_events_narrow(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(b"2") + con.out.write.assert_any_call(b"+") + con.out.write.assert_any_call(b"3") + con.out.write.assert_any_call(b"\\") + con.out.write.assert_any_call(b"\n") + con.out.write.assert_any_call(b"4") + con.restore() + + def test_resize_wider(self): + code = "1234567890" + events = code_to_events(code) + reader, console = self.handle_events_narrow(events) + + console.height = 20 + console.width = 80 + console.getheightwidth = MagicMock(lambda _: (20, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + + con.out.write.assert_any_call(self.move_right(2)) + con.out.write.assert_any_call(self.move_up(2)) + con.out.write.assert_any_call(b"567890") + + con.restore() + + def test_resize_narrower(self): + code = "1234567890" + events = code_to_events(code) + reader, console = self.handle_events(events) + + console.height = 20 + console.width = 4 + console.getheightwidth = MagicMock(lambda _: (20, 4)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + + con.out.write.assert_any_call(b"456\\") + con.out.write.assert_any_call(b"789\\") + + con.restore() + + def test_cursor_left(self): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_left()) + con.restore() + + def test_cursor_left_right(self): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_left()) + con.out.write.assert_any_call(self.move_right()) + con.restore() + + def test_cursor_up(self): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_up()) + con.restore() + + def test_cursor_up_down(self): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_up()) + con.out.write.assert_any_call(self.move_down()) + con.restore() + + def test_cursor_back_write(self): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(self.move_left()) + con.out.write.assert_any_call(b"21") + con.restore() + + def test_multiline_function_move_up_short_terminal(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, con = self.handle_events_short(events) + con.out.write.assert_any_call(self.move_left(5)) + con.out.write.assert_any_call(self.move_up()) + con.restore() + + def test_multiline_function_move_up_down_short_terminal(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, con = self.handle_events_short(events) + con.out.write.assert_any_call(self.move_left(8)) + con.out.write.assert_any_call(self.erase_in_line()) + con.restore() + + def test_resize_bigger_on_multiline_function(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = self.handle_events_short(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + con.out.write.assert_has_calls( + [ + call(self.move_left(5)), + call(self.move_up()), + call(b"def f():"), + call(self.move_left(3)), + call(self.move_down()), + ] + ) + console.restore() + con.restore() + + def test_resize_smaller_on_multiline_function(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = self.handle_events_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + con.out.write.assert_has_calls( + [ + call(self.move_left(5)), + call(self.move_up()), + call(self.erase_in_line()), + call(b" foo"), + ] + ) + console.restore() + con.restore() + + def move_up(self, lines=1): + return MOVE_UP.format(lines).encode("utf8") + + def move_down(self, lines=1): + return MOVE_DOWN.format(lines).encode("utf8") + + def move_left(self, cols=1): + return MOVE_LEFT.format(cols).encode("utf8") + + def move_right(self, cols=1): + return MOVE_RIGHT.format(cols).encode("utf8") + + def erase_in_line(self): + return ERASE_IN_LINE.encode("utf8") + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Windows/2024-05-25-18-43-10.gh-issue-111201.SLPJIx.rst b/Misc/NEWS.d/next/Windows/2024-05-25-18-43-10.gh-issue-111201.SLPJIx.rst new file mode 100644 index 00000000000000..f3918ed633d78c --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-05-25-18-43-10.gh-issue-111201.SLPJIx.rst @@ -0,0 +1 @@ +Add support for new pyrepl on Windows From 010aaa32fb93c5033a698d7213469af02d76fef3 Mon Sep 17 00:00:00 2001 From: Katie Bell Date: Fri, 31 May 2024 17:58:46 +1000 Subject: [PATCH 290/903] gh-97747: Improvements to WASM browser REPL. (#97665) Improvements to WASM browser REPL. Adds a text box to write and run code outside the REPL, a stop button, and handling of Ctrl-D for EOF. --- Tools/wasm/python.html | 67 ++++++++++++++++++++++++++++++++++--- Tools/wasm/python.worker.js | 10 ++++-- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/Tools/wasm/python.html b/Tools/wasm/python.html index 17ffa0ea8bfeff..81a035a5c4cd93 100644 --- a/Tools/wasm/python.html +++ b/Tools/wasm/python.html @@ -35,11 +35,12 @@

Simple REPL for Python WASM

-
+
+ +
+
The simple REPL provides a limited Python experience in the browser. diff --git a/Tools/wasm/python.worker.js b/Tools/wasm/python.worker.js index 1b794608fffe7b..4ce4e16fc0fa19 100644 --- a/Tools/wasm/python.worker.js +++ b/Tools/wasm/python.worker.js @@ -19,18 +19,18 @@ class StdinBuffer { } stdin = () => { - if (this.numberOfCharacters + 1 === this.readIndex) { + while (this.numberOfCharacters + 1 === this.readIndex) { if (!this.sentNull) { // Must return null once to indicate we're done for now. this.sentNull = true return null } this.sentNull = false + // Prompt will reset this.readIndex to 1 this.prompt() } const char = this.buffer[this.readIndex] this.readIndex += 1 - // How do I send an EOF?? return char } } @@ -71,7 +71,11 @@ var Module = { onmessage = (event) => { if (event.data.type === 'run') { - // TODO: Set up files from event.data.files + if (event.data.files) { + for (const [filename, contents] of Object.entries(event.data.files)) { + Module.FS.writeFile(filename, contents) + } + } const ret = callMain(event.data.args) postMessage({ type: 'finished', From b278c723d79a238b14e99908e83f4b1b6a39ed3d Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Fri, 31 May 2024 11:07:16 +0300 Subject: [PATCH 291/903] gh-119780: Adjust exception messages in Lib/test/test_format.py (GH-119781) Mismatches were just output to the stdout, without making the test failing. --- Lib/test/test_format.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 8cef621bd716ac..d2026152d8e747 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -304,9 +304,9 @@ def test_str_format(self): test_exc('%c', sys.maxunicode+1, OverflowError, "%c arg not in range(0x110000)") #test_exc('%c', 2**128, OverflowError, "%c arg not in range(0x110000)") - test_exc('%c', 3.14, TypeError, "%c requires int or char") - test_exc('%c', 'ab', TypeError, "%c requires int or char") - test_exc('%c', b'x', TypeError, "%c requires int or char") + test_exc('%c', 3.14, TypeError, "%c requires an int or a unicode character, not float") + test_exc('%c', 'ab', TypeError, "%c requires an int or a unicode character, not a string of length 2") + test_exc('%c', b'x', TypeError, "%c requires an int or a unicode character, not bytes") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: @@ -370,11 +370,11 @@ def __bytes__(self): test_exc(b"%c", 2**128, OverflowError, "%c arg not in range(256)") test_exc(b"%c", b"Za", TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not a bytes object of length 2") test_exc(b"%c", "Y", TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not str") test_exc(b"%c", 3.14, TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not float") test_exc(b"%b", "Xc", TypeError, "%b requires a bytes-like object, " "or an object that implements __bytes__, not 'str'") From 94e9585e99abc2d060cedc77b3c03e06b4a0a9c4 Mon Sep 17 00:00:00 2001 From: Christopher Chavez Date: Fri, 31 May 2024 03:23:53 -0500 Subject: [PATCH 292/903] =?UTF-8?q?gh-103194:=20Fix=20Tkinter=E2=80=99s=20?= =?UTF-8?q?Tcl=20value=20type=20handling=20for=20Tcl=208.7/9.0=20(GH-10384?= =?UTF-8?q?6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some of standard Tcl types were renamed, removed, or no longer registered in Tcl 8.7/9.0. This change fixes automatic conversion of Tcl values to Python values to avoid returning a Tcl_Obj where the primary Python types (int, bool, str, bytes) were returned in older Tcl. --- ...-04-24-05-34-23.gh-issue-103194.GwBwWL.rst | 4 ++ Modules/_tkinter.c | 50 ++++++++++++------- 2 files changed, 36 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst diff --git a/Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst b/Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst new file mode 100644 index 00000000000000..3f70168b81069e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst @@ -0,0 +1,4 @@ +Prepare Tkinter for C API changes in Tcl 8.7/9.0 to avoid +:class:`_tkinter.Tcl_Obj` being unexpectedly returned +instead of :class:`bool`, :class:`str`, +:class:`bytearray`, or :class:`int`. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index c7e271faa4cf34..0cff36dd307c39 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -318,6 +318,7 @@ typedef struct { const Tcl_ObjType *BignumType; const Tcl_ObjType *ListType; const Tcl_ObjType *StringType; + const Tcl_ObjType *UTF32StringType; } TkappObject; #define Tkapp_Interp(v) (((TkappObject *) (v))->interp) @@ -588,14 +589,40 @@ Tkapp_New(const char *screenName, const char *className, } v->OldBooleanType = Tcl_GetObjType("boolean"); - v->BooleanType = Tcl_GetObjType("booleanString"); - v->ByteArrayType = Tcl_GetObjType("bytearray"); + { + Tcl_Obj *value; + int boolValue; + + /* Tcl 8.5 "booleanString" type is not registered + and is renamed to "boolean" in Tcl 9.0. + Based on approach suggested at + https://core.tcl-lang.org/tcl/info/3bb3bcf2da5b */ + value = Tcl_NewStringObj("true", -1); + Tcl_GetBooleanFromObj(NULL, value, &boolValue); + v->BooleanType = value->typePtr; + Tcl_DecrRefCount(value); + + // "bytearray" type is not registered in Tcl 9.0 + value = Tcl_NewByteArrayObj(NULL, 0); + v->ByteArrayType = value->typePtr; + Tcl_DecrRefCount(value); + } v->DoubleType = Tcl_GetObjType("double"); + /* TIP 484 suggests retrieving the "int" type without Tcl_GetObjType("int") + since it is no longer registered in Tcl 9.0. But even though Tcl 8.7 + only uses the "wideInt" type on platforms with 32-bit long, it still has + a registered "int" type, which FromObj() should recognize just in case. */ v->IntType = Tcl_GetObjType("int"); + if (v->IntType == NULL) { + Tcl_Obj *value = Tcl_NewIntObj(0); + v->IntType = value->typePtr; + Tcl_DecrRefCount(value); + } v->WideIntType = Tcl_GetObjType("wideInt"); v->BignumType = Tcl_GetObjType("bignum"); v->ListType = Tcl_GetObjType("list"); v->StringType = Tcl_GetObjType("string"); + v->UTF32StringType = Tcl_GetObjType("utf32string"); /* Delete the 'exit' command, which can screw things up */ Tcl_DeleteCommand(v->interp, "exit"); @@ -1124,14 +1151,6 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) return PyFloat_FromDouble(value->internalRep.doubleValue); } - if (value->typePtr == tkapp->IntType) { - long longValue; - if (Tcl_GetLongFromObj(interp, value, &longValue) == TCL_OK) - return PyLong_FromLong(longValue); - /* If there is an error in the long conversion, - fall through to wideInt handling. */ - } - if (value->typePtr == tkapp->IntType || value->typePtr == tkapp->WideIntType) { result = fromWideIntObj(tkapp, value); @@ -1176,17 +1195,12 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) return result; } - if (value->typePtr == tkapp->StringType) { + if (value->typePtr == tkapp->StringType || + value->typePtr == tkapp->UTF32StringType) + { return unicodeFromTclObj(value); } - if (tkapp->BooleanType == NULL && - strcmp(value->typePtr->name, "booleanString") == 0) { - /* booleanString type is not registered in Tcl */ - tkapp->BooleanType = value->typePtr; - return fromBoolean(tkapp, value); - } - if (tkapp->BignumType == NULL && strcmp(value->typePtr->name, "bignum") == 0) { /* bignum type is not registered in Tcl */ From dae0375bd97f3821c5db1602a0653a3c5dc53c5b Mon Sep 17 00:00:00 2001 From: Arnon Yaari Date: Fri, 31 May 2024 12:02:54 +0300 Subject: [PATCH 293/903] gh-111201: Improve pyrepl auto indentation (#119606) - auto-indent when editing multi-line block - ignore comments --- Lib/_pyrepl/readline.py | 27 +++++++--- Lib/test/test_pyrepl/test_pyrepl.py | 81 ++++++++++++++++++++++++++++- Lib/test/test_pyrepl/test_reader.py | 4 +- 3 files changed, 101 insertions(+), 11 deletions(-) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 248f3854a29689..7d811bf41773fe 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -237,13 +237,24 @@ def _get_first_indentation(buffer: list[str]) -> str | None: return None -def _is_last_char_colon(buffer: list[str]) -> bool: - i = len(buffer) - while i > 0: - i -= 1 - if buffer[i] not in " \t\n": # ignore whitespaces - return buffer[i] == ":" - return False +def _should_auto_indent(buffer: list[str], pos: int) -> bool: + # check if last character before "pos" is a colon, ignoring + # whitespaces and comments. + last_char = None + while pos > 0: + pos -= 1 + if last_char is None: + if buffer[pos] not in " \t\n": # ignore whitespaces + last_char = buffer[pos] + else: + # even if we found a non-whitespace character before + # original pos, we keep going back until newline is reached + # to make sure we ignore comments + if buffer[pos] == "\n": + break + if buffer[pos] == "#": + last_char = None + return last_char == ":" class maybe_accept(commands.Command): @@ -280,7 +291,7 @@ def _newline_before_pos(): for i in range(prevlinestart, prevlinestart + indent): r.insert(r.buffer[i]) r.update_last_used_indentation() - if _is_last_char_colon(r.buffer): + if _should_auto_indent(r.buffer, r.pos): if r.last_used_indentation is not None: indentation = r.last_used_indentation else: diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index aa2722095794c9..45114e7315749f 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -312,6 +312,14 @@ def test_cursor_position_after_wrap_and_move_up(self): self.assertEqual(reader.pos, 10) self.assertEqual(reader.cxy, (1, 1)) + +class TestPyReplAutoindent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + def test_auto_indent_default(self): # fmt: off input_code = ( @@ -372,7 +380,6 @@ def test_auto_indent_prev_block(self): ), ) - output_code = ( "def g():\n" " pass\n" @@ -385,6 +392,78 @@ def test_auto_indent_prev_block(self): output2 = multiline_input(reader) self.assertEqual(output2, output_code) + def test_auto_indent_multiline(self): + # fmt: off + events = itertools.chain( + code_to_events( + "def f():\n" + "pass" + ), + [ + # go to the end of the first line + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # new line should be autoindented + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + code_to_events( + "pass" + ), + [ + # go to end of last line + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # double newline to terminate the block + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + output_code = ( + "def f():\n" + " pass\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_with_comment(self): + # fmt: off + events = code_to_events( + "def f(): # foo\n" + "pass\n\n" + ) + + output_code = ( + "def f(): # foo\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_ignore_comments(self): + # fmt: off + events = code_to_events( + "pass #:\n" + ) + + output_code = ( + "pass #:" + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + class TestPyReplOutput(TestCase): def prepare_reader(self, events): diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 7bf7a36d8d7bb9..c9b03d5e711539 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -168,8 +168,8 @@ def test_newline_within_block_trailing_whitespace(self): expected = ( "def foo():\n" - "\n" - "\n" + " \n" + " \n" " a = 1\n" " \n" " " # HistoricalReader will trim trailing whitespace From b9965ef282d6662145d2e05b080c811132ce6fde Mon Sep 17 00:00:00 2001 From: Joshua Herman <30265+zitterbewegung@users.noreply.github.com> Date: Fri, 31 May 2024 05:05:09 -0500 Subject: [PATCH 294/903] gh-119189: Fix the power operator for Fraction (GH-119242) When using the ** operator or pow() with Fraction as the base and an exponent that is not rational, a float, or a complex, the fraction is no longer converted to a float. --- Lib/fractions.py | 4 +++- Lib/test/test_fractions.py | 10 +++++----- Misc/ACKS | 1 + .../2024-05-20-13-48-37.gh-issue-119189.dhJVs5.rst | 3 +++ 4 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-20-13-48-37.gh-issue-119189.dhJVs5.rst diff --git a/Lib/fractions.py b/Lib/fractions.py index f91b4f35eff370..95adccd86e33a0 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -877,8 +877,10 @@ def __pow__(a, b, modulo=None): # A fractional power will generally produce an # irrational number. return float(a) ** float(b) - else: + elif isinstance(b, (float, complex)): return float(a) ** b + else: + return NotImplemented def __rpow__(b, a): """a ** b""" diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 28607ee37000f9..3c7780e40db096 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -925,21 +925,21 @@ def testMixedPower(self): self.assertTypedEquals(Root(4) ** F(2, 1), Root(4, F(1))) self.assertTypedEquals(Root(4) ** F(-2, 1), Root(4, -F(1))) self.assertTypedEquals(Root(4) ** F(-2, 3), Root(4, -3.0)) - self.assertEqual(F(3, 2) ** SymbolicReal('X'), SymbolicReal('1.5 ** X')) + self.assertEqual(F(3, 2) ** SymbolicReal('X'), SymbolicReal('3/2 ** X')) self.assertEqual(SymbolicReal('X') ** F(3, 2), SymbolicReal('X ** 1.5')) - self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(2.25, 0.0)) - self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(1.0, 0.0)) + self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(F(9,4), 0.0)) + self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(F(1), 0.0)) self.assertTypedEquals(F(3, 2) ** RectComplex(2, 0), Polar(2.25, 0.0)) self.assertTypedEquals(F(1, 1) ** RectComplex(2, 3), Polar(1.0, 0.0)) self.assertTypedEquals(Polar(4, 2) ** F(3, 2), Polar(8.0, 3.0)) self.assertTypedEquals(Polar(4, 2) ** F(3, 1), Polar(64, 6)) self.assertTypedEquals(Polar(4, 2) ** F(-3, 1), Polar(0.015625, -6)) self.assertTypedEquals(Polar(4, 2) ** F(-3, 2), Polar(0.125, -3.0)) - self.assertEqual(F(3, 2) ** SymbolicComplex('X'), SymbolicComplex('1.5 ** X')) + self.assertEqual(F(3, 2) ** SymbolicComplex('X'), SymbolicComplex('3/2 ** X')) self.assertEqual(SymbolicComplex('X') ** F(3, 2), SymbolicComplex('X ** 1.5')) - self.assertEqual(F(3, 2) ** Symbolic('X'), Symbolic('1.5 ** X')) + self.assertEqual(F(3, 2) ** Symbolic('X'), Symbolic('3/2 ** X')) self.assertEqual(Symbolic('X') ** F(3, 2), Symbolic('X ** 1.5')) def testMixingWithDecimal(self): diff --git a/Misc/ACKS b/Misc/ACKS index 9c10a76f1df624..2e7e12481bacd7 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -751,6 +751,7 @@ Kasun Herath Chris Herborth Ivan Herman Jürgen Hermann +Joshua Jay Herman Gary Herron Ernie Hershey Thomas Herve diff --git a/Misc/NEWS.d/next/Library/2024-05-20-13-48-37.gh-issue-119189.dhJVs5.rst b/Misc/NEWS.d/next/Library/2024-05-20-13-48-37.gh-issue-119189.dhJVs5.rst new file mode 100644 index 00000000000000..e5cfbcf95a0b81 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-20-13-48-37.gh-issue-119189.dhJVs5.rst @@ -0,0 +1,3 @@ +When using the ``**`` operator or :func:`pow` with :class:`~fractions.Fraction` +as the base and an exponent that is not rational, a float, or a complex, the +fraction is no longer converted to a float. From 0e8d35b931f41210483cc51c4169e9a943c7f166 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 31 May 2024 13:07:19 +0100 Subject: [PATCH 295/903] gh-116145: Updated bundled Tcl/Tk on Windows to 8.6.14 (GH-117030) --- ...024-03-19-19-04-56.gh-issue-116145.srVT3d.rst | 1 + Misc/externals.spdx.json | 16 ++++++++-------- PCbuild/get_externals.bat | 6 +++--- PCbuild/tcltk.props | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-03-19-19-04-56.gh-issue-116145.srVT3d.rst diff --git a/Misc/NEWS.d/next/Windows/2024-03-19-19-04-56.gh-issue-116145.srVT3d.rst b/Misc/NEWS.d/next/Windows/2024-03-19-19-04-56.gh-issue-116145.srVT3d.rst new file mode 100644 index 00000000000000..7f840b0556048a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-03-19-19-04-56.gh-issue-116145.srVT3d.rst @@ -0,0 +1 @@ +Updated bundled Tcl/Tk to 8.6.14. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 58f8e0afd71f1b..758d41910054ce 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -112,42 +112,42 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "1d3f2015e49e269cf681373d433cd54d88d5ef7443fe87f5f50f5fcfe9003e73" + "checksumValue": "ad7623a44e1b6e42df47ba8f16b2b0435ac605650b5054077c4355a30473074c" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.13.1.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.14.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.1:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.14.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tcl-core", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.13.1" + "versionInfo": "8.6.14.0" }, { "SPDXID": "SPDXRef-PACKAGE-tk", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "6056203b8a6aaf6ea89d90a7b55dc7f407e55c093f731a98fd830a712a3c81d3" + "checksumValue": "e8d5cbe97952037962518b69aba85e324d80aa189054c163ab0ee764a448e802" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.13.1.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.14.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.1:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.14.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tk", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.13.1" + "versionInfo": "8.6.14.0" }, { "SPDXID": "SPDXRef-PACKAGE-xz", diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 761d3de93b777d..1927938ef0821c 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -56,8 +56,8 @@ if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.45.3.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.14.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.14.0 set libraries=%libraries% xz-5.2.5 set libraries=%libraries% zlib-1.3.1 @@ -78,7 +78,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.13 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.1 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.14.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 8ddf01d5dd1dca..95b699b4cac0aa 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 8.6.13.1 + 8.6.14.0 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) From 4c387a76f3ac8509c29634f4bbda6c37a67550d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 31 May 2024 08:41:26 -0400 Subject: [PATCH 296/903] gh-111201: [pyrepl] Ensure optional platform-specific imports are optional (GH-119834) --- Lib/test/test_pyrepl/test_windows_console.py | 24 ++++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py index e87dfe99b1a17d..60392e231508b6 100644 --- a/Lib/test/test_pyrepl/test_windows_console.py +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -1,24 +1,24 @@ import itertools import sys import unittest -from _pyrepl.console import Event, Console -from _pyrepl.windows_console import ( - MOVE_LEFT, - MOVE_RIGHT, - MOVE_UP, - MOVE_DOWN, - ERASE_IN_LINE, -) + from functools import partial from typing import Iterable -from unittest import TestCase, main -from unittest.mock import MagicMock, call, patch, ANY +from unittest import TestCase +from unittest.mock import MagicMock, call from .support import handle_all_events, code_to_events try: - from _pyrepl.console import Event - from _pyrepl.windows_console import WindowsConsole + from _pyrepl.console import Event, Console + from _pyrepl.windows_console import ( + WindowsConsole, + MOVE_LEFT, + MOVE_RIGHT, + MOVE_UP, + MOVE_DOWN, + ERASE_IN_LINE, + ) except ImportError: pass From 91601a55964fdb3c02b21fa3c8dc629daff2390f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 31 May 2024 16:06:10 +0200 Subject: [PATCH 297/903] gh-111201: Skip pyrepl Windows tests earlier (#119848) Don't attempt to load pyrepl Windows console if platforms others than Windows. For example, the import can fail if ctypes is missing. --- Lib/test/test_pyrepl/test_windows_console.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py index 60392e231508b6..e52a54d31fb5d8 100644 --- a/Lib/test/test_pyrepl/test_windows_console.py +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -1,7 +1,11 @@ -import itertools import sys import unittest +if sys.platform != 'win32': + raise unittest.SkipTest("test only relevant on win32") + + +import itertools from functools import partial from typing import Iterable from unittest import TestCase @@ -23,7 +27,6 @@ pass -@unittest.skipIf(sys.platform != "win32", "Test class specifically for Windows") class WindowsConsoleTests(TestCase): def console(self, events, **kwargs) -> Console: console = WindowsConsole() From 891c1e36f4e08da107443772a4eb50c72a83836d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 31 May 2024 16:49:26 +0200 Subject: [PATCH 298/903] gh-119853: Add Include/refcount.h file (#119854) --- Include/Python.h | 1 + Include/object.h | 482 ------------------------------------------- Include/refcount.h | 500 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 501 insertions(+), 482 deletions(-) create mode 100644 Include/refcount.h diff --git a/Include/Python.h b/Include/Python.h index e05901b9e52b5a..502c5ec5aeaa3c 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -61,6 +61,7 @@ #include "pystats.h" #include "pyatomic.h" #include "object.h" +#include "refcount.h" #include "objimpl.h" #include "typeslots.h" #include "pyhash.h" diff --git a/Include/object.h b/Include/object.h index 9132784628a501..c8c63b9b2b1450 100644 --- a/Include/object.h +++ b/Include/object.h @@ -59,59 +59,6 @@ whose size is determined when the object is allocated. /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; -/* -Immortalization: - -The following indicates the immortalization strategy depending on the amount -of available bits in the reference count field. All strategies are backwards -compatible but the specific reference count value or immortalization check -might change depending on the specializations for the underlying system. - -Proper deallocation of immortal instances requires distinguishing between -statically allocated immortal instances vs those promoted by the runtime to be -immortal. The latter should be the only instances that require -cleanup during runtime finalization. -*/ - -#if SIZEOF_VOID_P > 4 -/* -In 64+ bit systems, an object will be marked as immortal by setting all of the -lower 32 bits of the reference count field, which is equal to: 0xFFFFFFFF - -Using the lower 32 bits makes the value backwards compatible by allowing -C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely -increase and decrease the objects reference count. The object would lose its -immortality, but the execution would still be correct. - -Reference count increases will use saturated arithmetic, taking advantage of -having all the lower 32 bits set, which will avoid the reference count to go -beyond the refcount limit. Immortality checks for reference count decreases will -be done by checking the bit sign flag in the lower 32 bits. -*/ -#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX) - -#else -/* -In 32 bit systems, an object will be marked as immortal by setting all of the -lower 30 bits of the reference count field, which is equal to: 0x3FFFFFFF - -Using the lower 30 bits makes the value backwards compatible by allowing -C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely -increase and decrease the objects reference count. The object would lose its -immortality, but the execution would still be correct. - -Reference count increases and decreases will first go through an immortality -check by comparing the reference count field to the immortality reference count. -*/ -#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX >> 2) -#endif - -// Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is -// always 32-bits. -#ifdef Py_GIL_DISABLED -#define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX -#endif - // Kept for backward compatibility. It was needed by Py_TRACE_REFS build. #define _PyObject_EXTRA_INIT @@ -190,20 +137,6 @@ struct _object { // fields have been merged. #define _Py_UNOWNED_TID 0 -// The shared reference count uses the two least-significant bits to store -// flags. The remaining bits are used to store the reference count. -#define _Py_REF_SHARED_SHIFT 2 -#define _Py_REF_SHARED_FLAG_MASK 0x3 - -// The shared flags are initialized to zero. -#define _Py_REF_SHARED_INIT 0x0 -#define _Py_REF_MAYBE_WEAKREF 0x1 -#define _Py_REF_QUEUED 0x2 -#define _Py_REF_MERGED 0x3 - -// Create a shared field from a refcnt and desired flags -#define _Py_REF_SHARED(refcnt, flags) (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) - // NOTE: In non-free-threaded builds, `struct _PyMutex` is defined in // pycore_lock.h. See pycore_lock.h for more details. struct _PyMutex { uint8_t v; }; @@ -311,24 +244,6 @@ _Py_IsOwnedByCurrentThread(PyObject *ob) } #endif -static inline Py_ssize_t Py_REFCNT(PyObject *ob) { -#if !defined(Py_GIL_DISABLED) - return ob->ob_refcnt; -#else - uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return _Py_IMMORTAL_REFCNT; - } - Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); - return _Py_STATIC_CAST(Py_ssize_t, local) + - Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); -#endif -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) -#endif - - // bpo-39573: The Py_SET_TYPE() function must be used to set an object type. static inline PyTypeObject* Py_TYPE(PyObject *ob) { return ob->ob_type; @@ -350,19 +265,6 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) { # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) #endif -static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) -{ -#if defined(Py_GIL_DISABLED) - return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == - _Py_IMMORTAL_REFCNT_LOCAL); -#elif SIZEOF_VOID_P > 4 - return (_Py_CAST(PY_INT32_T, op->ob_refcnt) < 0); -#else - return (op->ob_refcnt == _Py_IMMORTAL_REFCNT); -#endif -} -#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) - static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; } @@ -371,55 +273,6 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { #endif -// Py_SET_REFCNT() implementation for stable ABI -PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); - -static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { -#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 - // Stable ABI implements Py_SET_REFCNT() as a function call - // on limited C API version 3.13 and newer. - _Py_SetRefcnt(ob, refcnt); -#else - // This immortal check is for code that is unaware of immortal objects. - // The runtime tracks these objects and we should avoid as much - // as possible having extensions inadvertently change the refcnt - // of an immortalized object. - if (_Py_IsImmortal(ob)) { - return; - } - -#ifndef Py_GIL_DISABLED - ob->ob_refcnt = refcnt; -#else - if (_Py_IsOwnedByCurrentThread(ob)) { - if ((size_t)refcnt > (size_t)UINT32_MAX) { - // On overflow, make the object immortal - ob->ob_tid = _Py_UNOWNED_TID; - ob->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; - ob->ob_ref_shared = 0; - } - else { - // Set local refcount to desired refcount and shared refcount - // to zero, but preserve the shared refcount flags. - ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt); - ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK; - } - } - else { - // Set local refcount to zero and shared refcount to desired refcount. - // Mark the object as merged. - ob->ob_tid = _Py_UNOWNED_TID; - ob->ob_ref_local = 0; - ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); - } -#endif // Py_GIL_DISABLED -#endif // Py_LIMITED_API+0 < 0x030d0000 -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) -#endif - - static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } @@ -740,341 +593,6 @@ given type object has a specified feature. #define Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18) -/* -The macros Py_INCREF(op) and Py_DECREF(op) are used to increment or decrement -reference counts. Py_DECREF calls the object's deallocator function when -the refcount falls to 0; for -objects that don't contain references to other objects or heap memory -this can be the standard function free(). Both macros can be used -wherever a void expression is allowed. The argument must not be a -NULL pointer. If it may be NULL, use Py_XINCREF/Py_XDECREF instead. -The macro _Py_NewReference(op) initialize reference counts to 1, and -in special builds (Py_REF_DEBUG, Py_TRACE_REFS) performs additional -bookkeeping appropriate to the special build. - -We assume that the reference count field can never overflow; this can -be proven when the size of the field is the same as the pointer size, so -we ignore the possibility. Provided a C int is at least 32 bits (which -is implicitly assumed in many parts of this code), that's enough for -about 2**31 references to an object. - -XXX The following became out of date in Python 2.2, but I'm not sure -XXX what the full truth is now. Certainly, heap-allocated type objects -XXX can and should be deallocated. -Type objects should never be deallocated; the type pointer in an object -is not considered to be a reference to the type object, to save -complications in the deallocation function. (This is actually a -decision that's up to the implementer of each new type so if you want, -you can count such references to the type object.) -*/ - -#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) -PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, - PyObject *op); -PyAPI_FUNC(void) _Py_INCREF_IncRefTotal(void); -PyAPI_FUNC(void) _Py_DECREF_DecRefTotal(void); -#endif // Py_REF_DEBUG && !Py_LIMITED_API - -PyAPI_FUNC(void) _Py_Dealloc(PyObject *); - -/* -These are provided as conveniences to Python runtime embedders, so that -they can have object code that is not dependent on Python compilation flags. -*/ -PyAPI_FUNC(void) Py_IncRef(PyObject *); -PyAPI_FUNC(void) Py_DecRef(PyObject *); - -// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL. -// Private functions used by Py_INCREF() and Py_DECREF(). -PyAPI_FUNC(void) _Py_IncRef(PyObject *); -PyAPI_FUNC(void) _Py_DecRef(PyObject *); - -static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) -{ -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) - // Stable ABI implements Py_INCREF() as a function call on limited C API - // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() - // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. - // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. -# if Py_LIMITED_API+0 >= 0x030a00A7 - _Py_IncRef(op); -# else - Py_IncRef(op); -# endif -#else - // Non-limited C API and limited C API for Python 3.9 and older access - // directly PyObject.ob_refcnt. -#if defined(Py_GIL_DISABLED) - uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); - uint32_t new_local = local + 1; - if (new_local == 0) { - // local is equal to _Py_IMMORTAL_REFCNT: do nothing - return; - } - if (_Py_IsOwnedByCurrentThread(op)) { - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local); - } - else { - _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); - } -#elif SIZEOF_VOID_P > 4 - // Portable saturated add, branching on the carry flag and set low bits - PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; - PY_UINT32_T new_refcnt = cur_refcnt + 1; - if (new_refcnt == 0) { - // cur_refcnt is equal to _Py_IMMORTAL_REFCNT: the object is immortal, - // do nothing - return; - } - op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; -#else - // Explicitly check immortality against the immortal value - if (_Py_IsImmortal(op)) { - return; - } - op->ob_refcnt++; -#endif - _Py_INCREF_STAT_INC(); -#ifdef Py_REF_DEBUG - _Py_INCREF_IncRefTotal(); -#endif -#endif -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) -#endif - - -#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) -// Implements Py_DECREF on objects not owned by the current thread. -PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); -PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); - -// Called from Py_DECREF by the owning thread when the local refcount reaches -// zero. The call will deallocate the object if the shared refcount is also -// zero. Otherwise, the thread gives up ownership and merges the reference -// count fields. -PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); -#endif - -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) -// Stable ABI implements Py_DECREF() as a function call on limited C API -// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was -// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. -// Py_DecRef() accepts NULL whereas _Py_IncRef() doesn't. -static inline void Py_DECREF(PyObject *op) { -# if Py_LIMITED_API+0 >= 0x030a00A7 - _Py_DecRef(op); -# else - Py_DecRef(op); -# endif -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) - -#elif defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG) -static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) -{ - uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return; - } - _Py_DECREF_STAT_INC(); - _Py_DECREF_DecRefTotal(); - if (_Py_IsOwnedByCurrentThread(op)) { - if (local == 0) { - _Py_NegativeRefcount(filename, lineno, op); - } - local--; - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); - if (local == 0) { - _Py_MergeZeroLocalRefcount(op); - } - } - else { - _Py_DecRefSharedDebug(op, filename, lineno); - } -} -#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) - -#elif defined(Py_GIL_DISABLED) -static inline void Py_DECREF(PyObject *op) -{ - uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return; - } - _Py_DECREF_STAT_INC(); - if (_Py_IsOwnedByCurrentThread(op)) { - local--; - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); - if (local == 0) { - _Py_MergeZeroLocalRefcount(op); - } - } - else { - _Py_DecRefShared(op); - } -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) - -#elif defined(Py_REF_DEBUG) -static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) -{ - if (op->ob_refcnt <= 0) { - _Py_NegativeRefcount(filename, lineno, op); - } - if (_Py_IsImmortal(op)) { - return; - } - _Py_DECREF_STAT_INC(); - _Py_DECREF_DecRefTotal(); - if (--op->ob_refcnt == 0) { - _Py_Dealloc(op); - } -} -#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) - -#else -static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) -{ - // Non-limited C API and limited C API for Python 3.9 and older access - // directly PyObject.ob_refcnt. - if (_Py_IsImmortal(op)) { - return; - } - _Py_DECREF_STAT_INC(); - if (--op->ob_refcnt == 0) { - _Py_Dealloc(op); - } -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) -#endif - - -/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear - * and tp_dealloc implementations. - * - * Note that "the obvious" code can be deadly: - * - * Py_XDECREF(op); - * op = NULL; - * - * Typically, `op` is something like self->containee, and `self` is done - * using its `containee` member. In the code sequence above, suppose - * `containee` is non-NULL with a refcount of 1. Its refcount falls to - * 0 on the first line, which can trigger an arbitrary amount of code, - * possibly including finalizers (like __del__ methods or weakref callbacks) - * coded in Python, which in turn can release the GIL and allow other threads - * to run, etc. Such code may even invoke methods of `self` again, or cause - * cyclic gc to trigger, but-- oops! --self->containee still points to the - * object being torn down, and it may be in an insane state while being torn - * down. This has in fact been a rich historic source of miserable (rare & - * hard-to-diagnose) segfaulting (and other) bugs. - * - * The safe way is: - * - * Py_CLEAR(op); - * - * That arranges to set `op` to NULL _before_ decref'ing, so that any code - * triggered as a side-effect of `op` getting torn down no longer believes - * `op` points to a valid object. - * - * There are cases where it's safe to use the naive code, but they're brittle. - * For example, if `op` points to a Python integer, you know that destroying - * one of those can't cause problems -- but in part that relies on that - * Python integers aren't currently weakly referencable. Best practice is - * to use Py_CLEAR() even if you can't think of a reason for why you need to. - * - * gh-98724: Use a temporary variable to only evaluate the macro argument once, - * to avoid the duplication of side effects if the argument has side effects. - * - * gh-99701: If the PyObject* type is used with casting arguments to PyObject*, - * the code can be miscompiled with strict aliasing because of type punning. - * With strict aliasing, a compiler considers that two pointers of different - * types cannot read or write the same memory which enables optimization - * opportunities. - * - * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables, - * and so avoid type punning. Otherwise, use memcpy() which causes type erasure - * and so prevents the compiler to reuse an old cached 'op' value after - * Py_CLEAR(). - */ -#ifdef _Py_TYPEOF -#define Py_CLEAR(op) \ - do { \ - _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \ - _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \ - if (_tmp_old_op != NULL) { \ - *_tmp_op_ptr = _Py_NULL; \ - Py_DECREF(_tmp_old_op); \ - } \ - } while (0) -#else -#define Py_CLEAR(op) \ - do { \ - PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \ - PyObject *_tmp_old_op = (*_tmp_op_ptr); \ - if (_tmp_old_op != NULL) { \ - PyObject *_null_ptr = _Py_NULL; \ - memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \ - Py_DECREF(_tmp_old_op); \ - } \ - } while (0) -#endif - - -/* Function to use in case the object pointer can be NULL: */ -static inline void Py_XINCREF(PyObject *op) -{ - if (op != _Py_NULL) { - Py_INCREF(op); - } -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op)) -#endif - -static inline void Py_XDECREF(PyObject *op) -{ - if (op != _Py_NULL) { - Py_DECREF(op); - } -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op)) -#endif - -// Create a new strong reference to an object: -// increment the reference count of the object and return the object. -PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); - -// Similar to Py_NewRef(), but the object can be NULL. -PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); - -static inline PyObject* _Py_NewRef(PyObject *obj) -{ - Py_INCREF(obj); - return obj; -} - -static inline PyObject* _Py_XNewRef(PyObject *obj) -{ - Py_XINCREF(obj); - return obj; -} - -// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. -// Names overridden with macros by static inline functions for best -// performances. -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) -# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) -#else -# define Py_NewRef(obj) _Py_NewRef(obj) -# define Py_XNewRef(obj) _Py_XNewRef(obj) -#endif - - #define Py_CONSTANT_NONE 0 #define Py_CONSTANT_FALSE 1 #define Py_CONSTANT_TRUE 2 diff --git a/Include/refcount.h b/Include/refcount.h new file mode 100644 index 00000000000000..a0bd2087fb1b57 --- /dev/null +++ b/Include/refcount.h @@ -0,0 +1,500 @@ +#ifndef Py_REFCOUNT_H +#define Py_REFCOUNT_H +#ifdef __cplusplus +extern "C" { +#endif + + +/* +Immortalization: + +The following indicates the immortalization strategy depending on the amount +of available bits in the reference count field. All strategies are backwards +compatible but the specific reference count value or immortalization check +might change depending on the specializations for the underlying system. + +Proper deallocation of immortal instances requires distinguishing between +statically allocated immortal instances vs those promoted by the runtime to be +immortal. The latter should be the only instances that require +cleanup during runtime finalization. +*/ + +#if SIZEOF_VOID_P > 4 +/* +In 64+ bit systems, an object will be marked as immortal by setting all of the +lower 32 bits of the reference count field, which is equal to: 0xFFFFFFFF + +Using the lower 32 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. The object would lose its +immortality, but the execution would still be correct. + +Reference count increases will use saturated arithmetic, taking advantage of +having all the lower 32 bits set, which will avoid the reference count to go +beyond the refcount limit. Immortality checks for reference count decreases will +be done by checking the bit sign flag in the lower 32 bits. +*/ +#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX) + +#else +/* +In 32 bit systems, an object will be marked as immortal by setting all of the +lower 30 bits of the reference count field, which is equal to: 0x3FFFFFFF + +Using the lower 30 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. The object would lose its +immortality, but the execution would still be correct. + +Reference count increases and decreases will first go through an immortality +check by comparing the reference count field to the immortality reference count. +*/ +#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX >> 2) +#endif + +// Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is +// always 32-bits. +#ifdef Py_GIL_DISABLED +#define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX +#endif + + +#ifdef Py_GIL_DISABLED + // The shared reference count uses the two least-significant bits to store + // flags. The remaining bits are used to store the reference count. +# define _Py_REF_SHARED_SHIFT 2 +# define _Py_REF_SHARED_FLAG_MASK 0x3 + + // The shared flags are initialized to zero. +# define _Py_REF_SHARED_INIT 0x0 +# define _Py_REF_MAYBE_WEAKREF 0x1 +# define _Py_REF_QUEUED 0x2 +# define _Py_REF_MERGED 0x3 + + // Create a shared field from a refcnt and desired flags +# define _Py_REF_SHARED(refcnt, flags) \ + (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) +#endif // Py_GIL_DISABLED + + +static inline Py_ssize_t Py_REFCNT(PyObject *ob) { +#if !defined(Py_GIL_DISABLED) + return ob->ob_refcnt; +#else + uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return _Py_IMMORTAL_REFCNT; + } + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); + return _Py_STATIC_CAST(Py_ssize_t, local) + + Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); +#endif +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) +#endif + + +static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) +{ +#if defined(Py_GIL_DISABLED) + return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == + _Py_IMMORTAL_REFCNT_LOCAL); +#elif SIZEOF_VOID_P > 4 + return (_Py_CAST(PY_INT32_T, op->ob_refcnt) < 0); +#else + return (op->ob_refcnt == _Py_IMMORTAL_REFCNT); +#endif +} +#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) + + +// Py_SET_REFCNT() implementation for stable ABI +PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); + +static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 + // Stable ABI implements Py_SET_REFCNT() as a function call + // on limited C API version 3.13 and newer. + _Py_SetRefcnt(ob, refcnt); +#else + // This immortal check is for code that is unaware of immortal objects. + // The runtime tracks these objects and we should avoid as much + // as possible having extensions inadvertently change the refcnt + // of an immortalized object. + if (_Py_IsImmortal(ob)) { + return; + } + +#ifndef Py_GIL_DISABLED + ob->ob_refcnt = refcnt; +#else + if (_Py_IsOwnedByCurrentThread(ob)) { + if ((size_t)refcnt > (size_t)UINT32_MAX) { + // On overflow, make the object immortal + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + ob->ob_ref_shared = 0; + } + else { + // Set local refcount to desired refcount and shared refcount + // to zero, but preserve the shared refcount flags. + ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt); + ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK; + } + } + else { + // Set local refcount to zero and shared refcount to desired refcount. + // Mark the object as merged. + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = 0; + ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); + } +#endif // Py_GIL_DISABLED +#endif // Py_LIMITED_API+0 < 0x030d0000 +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) +#endif + + +/* +The macros Py_INCREF(op) and Py_DECREF(op) are used to increment or decrement +reference counts. Py_DECREF calls the object's deallocator function when +the refcount falls to 0; for +objects that don't contain references to other objects or heap memory +this can be the standard function free(). Both macros can be used +wherever a void expression is allowed. The argument must not be a +NULL pointer. If it may be NULL, use Py_XINCREF/Py_XDECREF instead. +The macro _Py_NewReference(op) initialize reference counts to 1, and +in special builds (Py_REF_DEBUG, Py_TRACE_REFS) performs additional +bookkeeping appropriate to the special build. + +We assume that the reference count field can never overflow; this can +be proven when the size of the field is the same as the pointer size, so +we ignore the possibility. Provided a C int is at least 32 bits (which +is implicitly assumed in many parts of this code), that's enough for +about 2**31 references to an object. + +XXX The following became out of date in Python 2.2, but I'm not sure +XXX what the full truth is now. Certainly, heap-allocated type objects +XXX can and should be deallocated. +Type objects should never be deallocated; the type pointer in an object +is not considered to be a reference to the type object, to save +complications in the deallocation function. (This is actually a +decision that's up to the implementer of each new type so if you want, +you can count such references to the type object.) +*/ + +#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) +PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, + PyObject *op); +PyAPI_FUNC(void) _Py_INCREF_IncRefTotal(void); +PyAPI_FUNC(void) _Py_DECREF_DecRefTotal(void); +#endif // Py_REF_DEBUG && !Py_LIMITED_API + +PyAPI_FUNC(void) _Py_Dealloc(PyObject *); + + +/* +These are provided as conveniences to Python runtime embedders, so that +they can have object code that is not dependent on Python compilation flags. +*/ +PyAPI_FUNC(void) Py_IncRef(PyObject *); +PyAPI_FUNC(void) Py_DecRef(PyObject *); + +// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL. +// Private functions used by Py_INCREF() and Py_DECREF(). +PyAPI_FUNC(void) _Py_IncRef(PyObject *); +PyAPI_FUNC(void) _Py_DecRef(PyObject *); + +static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) +{ +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) + // Stable ABI implements Py_INCREF() as a function call on limited C API + // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() + // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. + // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. +# if Py_LIMITED_API+0 >= 0x030a00A7 + _Py_IncRef(op); +# else + Py_IncRef(op); +# endif +#else + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. +#if defined(Py_GIL_DISABLED) + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + uint32_t new_local = local + 1; + if (new_local == 0) { + // local is equal to _Py_IMMORTAL_REFCNT: do nothing + return; + } + if (_Py_IsOwnedByCurrentThread(op)) { + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local); + } + else { + _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); + } +#elif SIZEOF_VOID_P > 4 + // Portable saturated add, branching on the carry flag and set low bits + PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; + PY_UINT32_T new_refcnt = cur_refcnt + 1; + if (new_refcnt == 0) { + // cur_refcnt is equal to _Py_IMMORTAL_REFCNT: the object is immortal, + // do nothing + return; + } + op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; +#else + // Explicitly check immortality against the immortal value + if (_Py_IsImmortal(op)) { + return; + } + op->ob_refcnt++; +#endif + _Py_INCREF_STAT_INC(); +#ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +#endif +#endif +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) +#endif + + +#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +// Implements Py_DECREF on objects not owned by the current thread. +PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); +PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); + +// Called from Py_DECREF by the owning thread when the local refcount reaches +// zero. The call will deallocate the object if the shared refcount is also +// zero. Otherwise, the thread gives up ownership and merges the reference +// count fields. +PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); +#endif + +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +// Stable ABI implements Py_DECREF() as a function call on limited C API +// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was +// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. +// Py_DecRef() accepts NULL whereas _Py_IncRef() doesn't. +static inline void Py_DECREF(PyObject *op) { +# if Py_LIMITED_API+0 >= 0x030a00A7 + _Py_DecRef(op); +# else + Py_DecRef(op); +# endif +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) + +#elif defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG) +static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) +{ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (_Py_IsOwnedByCurrentThread(op)) { + if (local == 0) { + _Py_NegativeRefcount(filename, lineno, op); + } + local--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); + if (local == 0) { + _Py_MergeZeroLocalRefcount(op); + } + } + else { + _Py_DecRefSharedDebug(op, filename, lineno); + } +} +#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + +#elif defined(Py_GIL_DISABLED) +static inline void Py_DECREF(PyObject *op) +{ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return; + } + _Py_DECREF_STAT_INC(); + if (_Py_IsOwnedByCurrentThread(op)) { + local--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); + if (local == 0) { + _Py_MergeZeroLocalRefcount(op); + } + } + else { + _Py_DecRefShared(op); + } +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) + +#elif defined(Py_REF_DEBUG) +static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) +{ + if (op->ob_refcnt <= 0) { + _Py_NegativeRefcount(filename, lineno, op); + } + if (_Py_IsImmortal(op)) { + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (--op->ob_refcnt == 0) { + _Py_Dealloc(op); + } +} +#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + +#else +static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) +{ + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. + if (_Py_IsImmortal(op)) { + return; + } + _Py_DECREF_STAT_INC(); + if (--op->ob_refcnt == 0) { + _Py_Dealloc(op); + } +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) +#endif + + +/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear + * and tp_dealloc implementations. + * + * Note that "the obvious" code can be deadly: + * + * Py_XDECREF(op); + * op = NULL; + * + * Typically, `op` is something like self->containee, and `self` is done + * using its `containee` member. In the code sequence above, suppose + * `containee` is non-NULL with a refcount of 1. Its refcount falls to + * 0 on the first line, which can trigger an arbitrary amount of code, + * possibly including finalizers (like __del__ methods or weakref callbacks) + * coded in Python, which in turn can release the GIL and allow other threads + * to run, etc. Such code may even invoke methods of `self` again, or cause + * cyclic gc to trigger, but-- oops! --self->containee still points to the + * object being torn down, and it may be in an insane state while being torn + * down. This has in fact been a rich historic source of miserable (rare & + * hard-to-diagnose) segfaulting (and other) bugs. + * + * The safe way is: + * + * Py_CLEAR(op); + * + * That arranges to set `op` to NULL _before_ decref'ing, so that any code + * triggered as a side-effect of `op` getting torn down no longer believes + * `op` points to a valid object. + * + * There are cases where it's safe to use the naive code, but they're brittle. + * For example, if `op` points to a Python integer, you know that destroying + * one of those can't cause problems -- but in part that relies on that + * Python integers aren't currently weakly referencable. Best practice is + * to use Py_CLEAR() even if you can't think of a reason for why you need to. + * + * gh-98724: Use a temporary variable to only evaluate the macro argument once, + * to avoid the duplication of side effects if the argument has side effects. + * + * gh-99701: If the PyObject* type is used with casting arguments to PyObject*, + * the code can be miscompiled with strict aliasing because of type punning. + * With strict aliasing, a compiler considers that two pointers of different + * types cannot read or write the same memory which enables optimization + * opportunities. + * + * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables, + * and so avoid type punning. Otherwise, use memcpy() which causes type erasure + * and so prevents the compiler to reuse an old cached 'op' value after + * Py_CLEAR(). + */ +#ifdef _Py_TYPEOF +#define Py_CLEAR(op) \ + do { \ + _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \ + _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + *_tmp_op_ptr = _Py_NULL; \ + Py_DECREF(_tmp_old_op); \ + } \ + } while (0) +#else +#define Py_CLEAR(op) \ + do { \ + PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \ + PyObject *_tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + PyObject *_null_ptr = _Py_NULL; \ + memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \ + Py_DECREF(_tmp_old_op); \ + } \ + } while (0) +#endif + + +/* Function to use in case the object pointer can be NULL: */ +static inline void Py_XINCREF(PyObject *op) +{ + if (op != _Py_NULL) { + Py_INCREF(op); + } +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op)) +#endif + +static inline void Py_XDECREF(PyObject *op) +{ + if (op != _Py_NULL) { + Py_DECREF(op); + } +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op)) +#endif + +// Create a new strong reference to an object: +// increment the reference count of the object and return the object. +PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); + +// Similar to Py_NewRef(), but the object can be NULL. +PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); + +static inline PyObject* _Py_NewRef(PyObject *obj) +{ + Py_INCREF(obj); + return obj; +} + +static inline PyObject* _Py_XNewRef(PyObject *obj) +{ + Py_XINCREF(obj); + return obj; +} + +// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. +// Names overridden with macros by static inline functions for best +// performances. +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) +# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) +#else +# define Py_NewRef(obj) _Py_NewRef(obj) +# define Py_XNewRef(obj) _Py_XNewRef(obj) +#endif + + +#ifdef __cplusplus +} +#endif +#endif // !Py_REFCOUNT_H From bcc1be39cb1d04ad9fc0bd1b9193d3972835a57c Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 31 May 2024 10:50:52 -0400 Subject: [PATCH 299/903] gh-119585: Fix crash involving `PyGILState_Release()` and `PyThreadState_Clear()` (#119753) Make sure that `gilstate_counter` is not zero in when calling `PyThreadState_Clear()`. A destructor called from `PyThreadState_Clear()` may call back into `PyGILState_Ensure()` and `PyGILState_Release()`. If `gilstate_counter` is zero, it will try to create a new thread state before the current active thread state is destroyed, leading to an assertion failure or crash. --- Lib/test/test_capi/test_misc.py | 16 ++++++++++++++++ ...024-05-29-21-05-59.gh-issue-119585.Sn7JL3.rst | 5 +++++ Modules/_testcapimodule.c | 9 +++++++++ Python/pystate.c | 6 ++++++ 4 files changed, 36 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2024-05-29-21-05-59.gh-issue-119585.Sn7JL3.rst diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index ed42d7b64302f9..f3d16e4a2fc92a 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2888,6 +2888,22 @@ def callback(): t.start() t.join() + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_thread_gilstate_in_clear(self): + # See https://github.com/python/cpython/issues/119585 + class C: + def __del__(self): + _testcapi.gilstate_ensure_release() + + # Thread-local variables are destroyed in `PyThreadState_Clear()`. + local_var = threading.local() + + def callback(): + local_var.x = C() + + _testcapi._test_thread_state(callback) + @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_gilstate_ensure_no_deadlock(self): diff --git a/Misc/NEWS.d/next/C API/2024-05-29-21-05-59.gh-issue-119585.Sn7JL3.rst b/Misc/NEWS.d/next/C API/2024-05-29-21-05-59.gh-issue-119585.Sn7JL3.rst new file mode 100644 index 00000000000000..038dec2dbf90d1 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-29-21-05-59.gh-issue-119585.Sn7JL3.rst @@ -0,0 +1,5 @@ +Fix crash when a thread state that was created by :c:func:`PyGILState_Ensure` +calls a destructor that during :c:func:`PyThreadState_Clear` that +calls back into :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`. +This might occur when in the free-threaded build or when using thread-local +variables whose destructors call :c:func:`PyGILState_Ensure`. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f99ebf0dde4f9e..b58c17260626c2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -764,6 +764,14 @@ test_thread_state(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +gilstate_ensure_release(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyGILState_STATE state = PyGILState_Ensure(); + PyGILState_Release(state); + Py_RETURN_NONE; +} + #ifndef MS_WINDOWS static PyThread_type_lock wait_done = NULL; @@ -3351,6 +3359,7 @@ static PyMethodDef TestMethods[] = { {"test_get_type_dict", test_get_type_dict, METH_NOARGS}, {"test_reftracer", test_reftracer, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, + {"gilstate_ensure_release", gilstate_ensure_release, METH_NOARGS}, #ifndef MS_WINDOWS {"_spawn_pthread_waiter", spawn_pthread_waiter, METH_NOARGS}, {"_end_spawned_pthread", end_spawned_pthread, METH_NOARGS}, diff --git a/Python/pystate.c b/Python/pystate.c index 1ea1ad982a0ec9..ad7e082ce0d37e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2808,12 +2808,18 @@ PyGILState_Release(PyGILState_STATE oldstate) /* can't have been locked when we created it */ assert(oldstate == PyGILState_UNLOCKED); // XXX Unbind tstate here. + // gh-119585: `PyThreadState_Clear()` may call destructors that + // themselves use PyGILState_Ensure and PyGILState_Release, so make + // sure that gilstate_counter is not zero when calling it. + ++tstate->gilstate_counter; PyThreadState_Clear(tstate); + --tstate->gilstate_counter; /* Delete the thread-state. Note this releases the GIL too! * It's vital that the GIL be held here, to avoid shutdown * races; see bugs 225673 and 1061968 (that nasty bug has a * habit of coming back). */ + assert(tstate->gilstate_counter == 0); assert(current_fast_get() == tstate); _PyThreadState_DeleteCurrent(tstate); } From 64ff1e217d963b48140326e8b63c62f4b306f4a0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 31 May 2024 17:18:40 +0200 Subject: [PATCH 300/903] gh-119770: Make termios ioctl() constants positive (#119840) --- Lib/test/test_ioctl.py | 22 ++++++------------- Lib/test/test_termios.py | 9 ++++++++ ...-05-31-12-57-31.gh-issue-119770.NCtels.rst | 1 + Modules/termios.c | 18 ++++++++++++--- 4 files changed, 32 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-31-12-57-31.gh-issue-119770.NCtels.rst diff --git a/Lib/test/test_ioctl.py b/Lib/test/test_ioctl.py index 7b7067eb7b61d4..04934dfa16a5f0 100644 --- a/Lib/test/test_ioctl.py +++ b/Lib/test/test_ioctl.py @@ -66,23 +66,15 @@ def test_ioctl_mutate_2048(self): # Test with a larger buffer, just for the record. self._check_ioctl_mutate_len(2048) - def test_ioctl_signed_unsigned_code_param(self): - if not pty: - raise unittest.SkipTest('pty module required') + @unittest.skipIf(pty is None, 'pty module required') + def test_ioctl_set_window_size(self): mfd, sfd = pty.openpty() try: - if termios.TIOCSWINSZ < 0: - set_winsz_opcode_maybe_neg = termios.TIOCSWINSZ - set_winsz_opcode_pos = termios.TIOCSWINSZ & 0xffffffff - else: - set_winsz_opcode_pos = termios.TIOCSWINSZ - set_winsz_opcode_maybe_neg, = struct.unpack("i", - struct.pack("I", termios.TIOCSWINSZ)) - - our_winsz = struct.pack("HHHH",80,25,0,0) - # test both with a positive and potentially negative ioctl code - new_winsz = fcntl.ioctl(mfd, set_winsz_opcode_pos, our_winsz) - new_winsz = fcntl.ioctl(mfd, set_winsz_opcode_maybe_neg, our_winsz) + # (rows, columns, xpixel, ypixel) + our_winsz = struct.pack("HHHH", 20, 40, 0, 0) + result = fcntl.ioctl(mfd, termios.TIOCSWINSZ, our_winsz) + new_winsz = struct.unpack("HHHH", result) + self.assertEqual(new_winsz[:2], (20, 40)) finally: os.close(mfd) os.close(sfd) diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py index 58698ffac2d981..22e397c7a409c4 100644 --- a/Lib/test/test_termios.py +++ b/Lib/test/test_termios.py @@ -211,6 +211,15 @@ def test_constants(self): self.assertLess(termios.VTIME, termios.NCCS) self.assertLess(termios.VMIN, termios.NCCS) + def test_ioctl_constants(self): + # gh-119770: ioctl() constants must be positive + for name in dir(termios): + if not name.startswith('TIO'): + continue + value = getattr(termios, name) + with self.subTest(name=name): + self.assertGreaterEqual(value, 0) + def test_exception(self): self.assertTrue(issubclass(termios.error, Exception)) self.assertFalse(issubclass(termios.error, OSError)) diff --git a/Misc/NEWS.d/next/Library/2024-05-31-12-57-31.gh-issue-119770.NCtels.rst b/Misc/NEWS.d/next/Library/2024-05-31-12-57-31.gh-issue-119770.NCtels.rst new file mode 100644 index 00000000000000..94265e442db584 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-31-12-57-31.gh-issue-119770.NCtels.rst @@ -0,0 +1 @@ +Make :mod:`termios` ``ioctl()`` constants positive. Patch by Victor Stinner. diff --git a/Modules/termios.c b/Modules/termios.c index 0633d8f82cc7e4..efb5fcc17fa5ef 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -1352,9 +1352,21 @@ termios_exec(PyObject *mod) } while (constant->name != NULL) { - if (PyModule_AddIntConstant( - mod, constant->name, constant->value) < 0) { - return -1; + if (strncmp(constant->name, "TIO", 3) == 0) { + // gh-119770: Convert value to unsigned int for ioctl() constants, + // constants can be negative on macOS whereas ioctl() expects an + // unsigned long 'request'. + unsigned int value = constant->value & UINT_MAX; + if (PyModule_Add(mod, constant->name, + PyLong_FromUnsignedLong(value)) < 0) { + return -1; + } + } + else { + if (PyModule_AddIntConstant( + mod, constant->name, constant->value) < 0) { + return -1; + } } ++constant; } From 078b8c8cf2bf68f7484cc4d2e3dd74b6fab55664 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 31 May 2024 13:04:59 -0400 Subject: [PATCH 301/903] gh-119369: Fix deadlock during thread exit in free-threaded build (#119528) Release the GIL before calling `_Py_qsbr_unregister`. The deadlock could occur when the GIL was enabled at runtime. The `_Py_qsbr_unregister` call might block while holding the GIL because the thread state was not active, but the GIL was still held. --- ...-05-24-21-16-52.gh-issue-119369.qBThho.rst | 2 ++ Python/pystate.c | 21 +++++++++++-------- Python/qsbr.c | 5 +++++ 3 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst new file mode 100644 index 00000000000000..7abdd5cd85ccd6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst @@ -0,0 +1,2 @@ +Fix deadlock during thread deletion in free-threaded build, which could +occur when the GIL was enabled at runtime. diff --git a/Python/pystate.c b/Python/pystate.c index ad7e082ce0d37e..36e4206b4a282e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1751,7 +1751,7 @@ decrement_stoptheworld_countdown(struct _stoptheworld_state *stw); /* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */ static void -tstate_delete_common(PyThreadState *tstate) +tstate_delete_common(PyThreadState *tstate, int release_gil) { assert(tstate->_status.cleared && !tstate->_status.finalized); tstate_verify_not_active(tstate); @@ -1793,10 +1793,6 @@ tstate_delete_common(PyThreadState *tstate) HEAD_UNLOCK(runtime); -#ifdef Py_GIL_DISABLED - _Py_qsbr_unregister(tstate); -#endif - // XXX Unbind in PyThreadState_Clear(), or earlier // (and assert not-equal here)? if (tstate->_status.bound_gilstate) { @@ -1807,6 +1803,14 @@ tstate_delete_common(PyThreadState *tstate) // XXX Move to PyThreadState_Clear()? clear_datastack(tstate); + if (release_gil) { + _PyEval_ReleaseLock(tstate->interp, tstate, 1); + } + +#ifdef Py_GIL_DISABLED + _Py_qsbr_unregister(tstate); +#endif + tstate->_status.finalized = 1; } @@ -1818,7 +1822,7 @@ zapthreads(PyInterpreterState *interp) when the threads are all really dead (XXX famous last words). */ while ((tstate = interp->threads.head) != NULL) { tstate_verify_not_active(tstate); - tstate_delete_common(tstate); + tstate_delete_common(tstate, 0); free_threadstate((_PyThreadStateImpl *)tstate); } } @@ -1829,7 +1833,7 @@ PyThreadState_Delete(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); tstate_verify_not_active(tstate); - tstate_delete_common(tstate); + tstate_delete_common(tstate, 0); free_threadstate((_PyThreadStateImpl *)tstate); } @@ -1842,8 +1846,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); #endif current_fast_clear(tstate->interp->runtime); - tstate_delete_common(tstate); - _PyEval_ReleaseLock(tstate->interp, tstate, 1); + tstate_delete_common(tstate, 1); // release GIL as part of call free_threadstate((_PyThreadStateImpl *)tstate); } diff --git a/Python/qsbr.c b/Python/qsbr.c index 1e02ff9c2e45f0..9cbce9044e2941 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -236,6 +236,11 @@ _Py_qsbr_unregister(PyThreadState *tstate) struct _qsbr_shared *shared = &tstate->interp->qsbr; struct _PyThreadStateImpl *tstate_imp = (_PyThreadStateImpl*) tstate; + // gh-119369: GIL must be released (if held) to prevent deadlocks, because + // we might not have an active tstate, which means taht blocking on PyMutex + // locks will not implicitly release the GIL. + assert(!tstate->_status.holds_gil); + PyMutex_Lock(&shared->mutex); // NOTE: we must load (or reload) the thread state's qbsr inside the mutex // because the array may have been resized (changing tstate->qsbr) while From 015b1fdd0ae03f94a5dfda051b020810d1c952dd Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 31 May 2024 18:09:48 +0100 Subject: [PATCH 302/903] gh-100117: Fix inaccuracy in documentation of the CodeObject's co_positions field. (#119364) --- Doc/reference/datamodel.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 0fe9681f93f135..134385ed2f1860 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1243,7 +1243,7 @@ Methods on code objects The iterator returns :class:`tuple`\s containing the ``(start_line, end_line, start_column, end_column)``. The *i-th* tuple corresponds to the - position of the source code that compiled to the *i-th* instruction. + position of the source code that compiled to the *i-th* code unit. Column information is 0-indexed utf-8 byte offsets on the given source line. From f3fc800d5f17b144a752a262102b750bedcdaa14 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Fri, 31 May 2024 12:19:54 -0500 Subject: [PATCH 303/903] contextlib docs: Clean up redundant 'up' after 'cleanup' (GH-119867) Reported by Michael Kass on docs@ --- Doc/library/contextlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 73e53aec9cbf1c..bad9da52d6a6ca 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -796,7 +796,7 @@ executing that callback:: if result: stack.pop_all() -This allows the intended cleanup up behaviour to be made explicit up front, +This allows the intended cleanup behaviour to be made explicit up front, rather than requiring a separate flag variable. If a particular application uses this pattern a lot, it can be simplified From 9bc6045842ebc91ec48ab163a9e1e8644231607c Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 31 May 2024 13:23:29 -0400 Subject: [PATCH 304/903] doc: Add glossary entry for "free threading" (#119865) --- Doc/glossary.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 1e5bafce861e52..ae9949bc2867c4 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -438,6 +438,12 @@ Glossary division. Note that ``(-11) // 4`` is ``-3`` because that is ``-2.75`` rounded *downward*. See :pep:`238`. + free threading + A threading model where multiple threads can run Python bytecode + simultaneously within the same interpreter. This is in contrast to + the :term:`global interpreter lock` which allows only one thread to + execute Python bytecode at a time. See :pep:`703`. + function A series of statements which returns some value to a caller. It can also be passed zero or more :term:`arguments ` which may be used in From 879d43b705faab0c59f1a6a0042e286f39f3a4ef Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 31 May 2024 14:18:24 -0400 Subject: [PATCH 305/903] gh-119799: Add missing `_Py_IncRefTotal` to `_Py_NewRefWithLock` (#119800) The free-threaded refleak builds were reporting negative refcount deltas in some tests because of a missing `_Py_NewRefWithLock`. --- Include/internal/pycore_object.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7602248f956405..f63e1da6fba025 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -497,6 +497,9 @@ _Py_NewRefWithLock(PyObject *op) if (_Py_TryIncrefFast(op)) { return op; } +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif _Py_INCREF_STAT_INC(); for (;;) { Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); From f9d47fed9fbbe9313404838050f6dfe1c7fe6340 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 31 May 2024 21:21:30 +0200 Subject: [PATCH 306/903] gh-119853: Add Include/refcount.h to projects (#119860) --- Makefile.pre.in | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ 3 files changed, 5 insertions(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index a3fca80d4448ca..9a2fc34f030662 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1055,6 +1055,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/pythread.h \ $(srcdir)/Include/pytypedefs.h \ $(srcdir)/Include/rangeobject.h \ + $(srcdir)/Include/refcount.h \ $(srcdir)/Include/setobject.h \ $(srcdir)/Include/sliceobject.h \ $(srcdir)/Include/structmember.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 16fb424b11c6a8..96960f0579a936 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -349,6 +349,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index cf9bc0f4bc1c70..2e4bd786be47e9 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -207,6 +207,9 @@ Include + + Include + Include From 2237946af0981c46dc7d3886477e425ccfb37f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 31 May 2024 16:26:02 -0400 Subject: [PATCH 307/903] gh-118894: Make asyncio REPL use pyrepl (GH-119433) --- Lib/_pyrepl/commands.py | 5 ++ Lib/_pyrepl/console.py | 57 +++++++++++- Lib/_pyrepl/reader.py | 1 + Lib/_pyrepl/simple_interact.py | 53 ++--------- Lib/asyncio/__main__.py | 89 +++++++++++++++---- Lib/test/test_pyrepl/test_interact.py | 2 +- ...-05-22-21-20-43.gh-issue-118894.xHdxR_.rst | 1 + 7 files changed, 143 insertions(+), 65 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index ed977f84baac4e..2ef5dada9d9e58 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -219,6 +219,11 @@ def do(self) -> None: os.kill(os.getpid(), signal.SIGINT) +class ctrl_c(Command): + def do(self) -> None: + raise KeyboardInterrupt + + class suspend(Command): def do(self) -> None: import signal diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index fcabf785069ecb..aa0bde865825c9 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -19,10 +19,14 @@ from __future__ import annotations -import sys +import _colorize # type: ignore[import-not-found] from abc import ABC, abstractmethod +import ast +import code from dataclasses import dataclass, field +import os.path +import sys TYPE_CHECKING = False @@ -136,3 +140,54 @@ def wait(self) -> None: @abstractmethod def repaint(self) -> None: ... + + +class InteractiveColoredConsole(code.InteractiveConsole): + def __init__( + self, + locals: dict[str, object] | None = None, + filename: str = "", + *, + local_exit: bool = False, + ) -> None: + super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + self.can_colorize = _colorize.can_colorize() + + def showsyntaxerror(self, filename=None): + super().showsyntaxerror(colorize=self.can_colorize) + + def showtraceback(self): + super().showtraceback(colorize=self.can_colorize) + + def runsource(self, source, filename="", symbol="single"): + try: + tree = ast.parse(source) + except (SyntaxError, OverflowError, ValueError): + self.showsyntaxerror(filename) + return False + if tree.body: + *_, last_stmt = tree.body + for stmt in tree.body: + wrapper = ast.Interactive if stmt is last_stmt else ast.Module + the_symbol = symbol if stmt is last_stmt else "exec" + item = wrapper([stmt]) + try: + code = self.compile.compiler(item, filename, the_symbol, dont_inherit=True) + except SyntaxError as e: + if e.args[0] == "'await' outside function": + python = os.path.basename(sys.executable) + e.add_note( + f"Try the asyncio REPL ({python} -m asyncio) to use" + f" top-level 'await' and run background asyncio tasks." + ) + self.showsyntaxerror(filename) + return False + except (OverflowError, ValueError): + self.showsyntaxerror(filename) + return False + + if code is None: + return True + + self.runcode(code) + return False diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 0045425cdddb79..5401ae7b0ae32d 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -131,6 +131,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: ("\\\\", "self-insert"), (r"\x1b[200~", "enable_bracketed_paste"), (r"\x1b[201~", "disable_bracketed_paste"), + (r"\x03", "ctrl-c"), ] + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index c624f6e12a7094..256bbc7c6d7626 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -25,14 +25,13 @@ from __future__ import annotations -import _colorize # type: ignore[import-not-found] import _sitebuiltins import linecache import sys import code -import ast from types import ModuleType +from .console import InteractiveColoredConsole from .readline import _get_reader, multiline_input _error: tuple[type[Exception], ...] | type[Exception] @@ -74,57 +73,21 @@ def _clear_screen(): "clear": _clear_screen, } -class InteractiveColoredConsole(code.InteractiveConsole): - def __init__( - self, - locals: dict[str, object] | None = None, - filename: str = "", - *, - local_exit: bool = False, - ) -> None: - super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] - self.can_colorize = _colorize.can_colorize() - - def showsyntaxerror(self, filename=None): - super().showsyntaxerror(colorize=self.can_colorize) - - def showtraceback(self): - super().showtraceback(colorize=self.can_colorize) - - def runsource(self, source, filename="", symbol="single"): - try: - tree = ast.parse(source) - except (OverflowError, SyntaxError, ValueError): - self.showsyntaxerror(filename) - return False - if tree.body: - *_, last_stmt = tree.body - for stmt in tree.body: - wrapper = ast.Interactive if stmt is last_stmt else ast.Module - the_symbol = symbol if stmt is last_stmt else "exec" - item = wrapper([stmt]) - try: - code = compile(item, filename, the_symbol, dont_inherit=True) - except (OverflowError, ValueError, SyntaxError): - self.showsyntaxerror(filename) - return False - - if code is None: - return True - - self.runcode(code) - return False - def run_multiline_interactive_console( - mainmodule: ModuleType | None= None, future_flags: int = 0 + mainmodule: ModuleType | None = None, + future_flags: int = 0, + console: code.InteractiveConsole | None = None, ) -> None: import __main__ from .readline import _setup _setup() mainmodule = mainmodule or __main__ - console = InteractiveColoredConsole(mainmodule.__dict__, filename="") + if console is None: + console = InteractiveColoredConsole( + mainmodule.__dict__, filename="" + ) if future_flags: console.compile.compiler.flags |= future_flags diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 9041b8b8316c1e..91fff9aaee337b 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,42 +1,49 @@ import ast import asyncio -import code import concurrent.futures import inspect +import os import site import sys import threading import types import warnings +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] +from _pyrepl.console import InteractiveColoredConsole + from . import futures -class AsyncIOInteractiveConsole(code.InteractiveConsole): +class AsyncIOInteractiveConsole(InteractiveColoredConsole): def __init__(self, locals, loop): - super().__init__(locals) + super().__init__(locals, filename="") self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT self.loop = loop def runcode(self, code): + global return_code future = concurrent.futures.Future() def callback(): + global return_code global repl_future - global repl_future_interrupted + global keyboard_interrupted repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False func = types.FunctionType(code, self.locals) try: coro = func() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except KeyboardInterrupt as ex: - repl_future_interrupted = True + keyboard_interrupted = True future.set_exception(ex) return except BaseException as ex: @@ -57,10 +64,12 @@ def callback(): try: return future.result() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except BaseException: - if repl_future_interrupted: + if keyboard_interrupted: self.write("\nKeyboardInterrupt\n") else: self.showtraceback() @@ -69,18 +78,56 @@ def callback(): class REPLThread(threading.Thread): def run(self): + global return_code + try: banner = ( f'asyncio REPL {sys.version} on {sys.platform}\n' f'Use "await" directly instead of "asyncio.run()".\n' f'Type "help", "copyright", "credits" or "license" ' f'for more information.\n' - f'{getattr(sys, "ps1", ">>> ")}import asyncio' ) - console.interact( - banner=banner, - exitmsg='exiting asyncio REPL...') + console.write(banner) + + if startup_path := os.getenv("PYTHONSTARTUP"): + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, console.locals) + + ps1 = getattr(sys, "ps1", ">>> ") + if can_colorize(): + ps1 = f"{ANSIColors.BOLD_MAGENTA}{ps1}{ANSIColors.RESET}" + console.write(f"{ps1}import asyncio\n") + + try: + import errno + if os.getenv("PYTHON_BASIC_REPL"): + raise RuntimeError("user environment requested basic REPL") + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + + # This import will fail on operating systems with no termios. + from _pyrepl.simple_interact import ( + check, + run_multiline_interactive_console, + ) + if err := check(): + raise RuntimeError(err) + except Exception as e: + console.interact(banner="", exitmsg=exit_message) + else: + try: + run_multiline_interactive_console(console=console) + except SystemExit: + # expected via the `exit` and `quit` commands + pass + except BaseException: + # unexpected issue + console.showtraceback() + console.write("Internal error, ") + return_code = 1 finally: warnings.filterwarnings( 'ignore', @@ -91,6 +138,9 @@ def run(self): if __name__ == '__main__': + CAN_USE_PYREPL = True + + return_code = 0 loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -103,7 +153,7 @@ def run(self): console = AsyncIOInteractiveConsole(repl_locals, loop) repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False try: import readline # NoQA @@ -126,7 +176,7 @@ def run(self): completer = rlcompleter.Completer(console.locals) readline.set_completer(completer.complete) - repl_thread = REPLThread() + repl_thread = REPLThread(name="Interactive thread") repl_thread.daemon = True repl_thread.start() @@ -134,9 +184,12 @@ def run(self): try: loop.run_forever() except KeyboardInterrupt: + keyboard_interrupted = True if repl_future and not repl_future.done(): repl_future.cancel() - repl_future_interrupted = True continue else: break + + console.write('exiting asyncio REPL...\n') + sys.exit(return_code) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index 4d01ea7620109d..df97b1354a168e 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -6,7 +6,7 @@ from test.support import force_not_colorized -from _pyrepl.simple_interact import InteractiveColoredConsole +from _pyrepl.console import InteractiveColoredConsole class TestSimpleInteract(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst b/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst new file mode 100644 index 00000000000000..ffc4ae336dc54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst @@ -0,0 +1 @@ +:mod:`asyncio` REPL now has the same capabilities as PyREPL. From 80a4e3899420faaa012c82b4e82cdb6675a6a944 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 31 May 2024 14:05:24 -0700 Subject: [PATCH 308/903] gh-119821: Support non-dict globals in LOAD_FROM_DICT_OR_GLOBALS (#119822) Support non-dict globals in LOAD_FROM_DICT_OR_GLOBALS The implementation basically copies LOAD_GLOBAL. Possibly it could be deduplicated, but that seems like it may get hairy since the two operations have different operands. This is important to fix in 3.14 for PEP 649, but it's a bug in earlier versions too, and we should backport to 3.13 and 3.12 if possible. --- Include/internal/pycore_opcode_metadata.h | 1 - Include/internal/pycore_uop_metadata.h | 4 --- Lib/test/test_type_aliases.py | 20 +++++++++++ ...-05-30-23-01-00.gh-issue-119821.jPGfvt.rst | 2 ++ Python/bytecodes.c | 35 ++++++++++++++----- Python/executor_cases.c.h | 30 +--------------- Python/generated_cases.c.h | 35 ++++++++++++++----- Python/optimizer_cases.c.h | 7 +--- 8 files changed, 76 insertions(+), 58 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d3535800139a66..0b835230974e39 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1323,7 +1323,6 @@ _PyOpcode_macro_expansion[256] = { [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { _LOAD_FAST_CHECK, 0, 0 } } }, [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _LOAD_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, - [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 78f0eafaa32042..690ae34a6eef98 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -107,7 +107,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, @@ -439,7 +438,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", - [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", @@ -692,8 +690,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_LOCALS: return 0; - case _LOAD_FROM_DICT_OR_GLOBALS: - return 1; case _LOAD_GLOBAL: return 0; case _GUARD_GLOBALS_VERSION: diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 9c325bc595f585..f8b395fdc8bb1d 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -1,4 +1,5 @@ import pickle +import textwrap import types import unittest from test.support import check_syntax_error, run_code @@ -328,3 +329,22 @@ def test_pickling_local(self): with self.subTest(thing=thing, proto=proto): with self.assertRaises(pickle.PickleError): pickle.dumps(thing, protocol=proto) + + +class TypeParamsExoticGlobalsTest(unittest.TestCase): + def test_exec_with_unusual_globals(self): + class customdict(dict): + def __missing__(self, key): + return key + + code = compile("type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["Alias"] + self.assertEqual(Alias.__value__, "undefined") + + code = compile("class A: type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["A"].Alias + self.assertEqual(Alias.__value__, "undefined") diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst new file mode 100644 index 00000000000000..cc25eee6dd6ae4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst @@ -0,0 +1,2 @@ +Fix execution of :ref:`annotation scopes ` within classes +when ``globals`` is set to a non-dict. Patch by Jelle Zijlstra. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9a8198515dea5e..1c12e1cddbbc10 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1385,18 +1385,35 @@ dummy_func( ERROR_NO_POP(); } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - ERROR_NO_POP(); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (v == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } ERROR_NO_POP(); } + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0, error); if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - ERROR_NO_POP(); + /* namespace 2: builtins */ + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0, error); + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + ERROR_IF(true, error); + } } } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e862364cb23e7a..0dfe490cb37047 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1405,35 +1405,7 @@ break; } - case _LOAD_FROM_DICT_OR_GLOBALS: { - PyObject *mod_or_class_dict; - PyObject *v; - oparg = CURRENT_OPARG(); - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - JUMP_TO_ERROR(); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - JUMP_TO_ERROR(); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - JUMP_TO_ERROR(); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - JUMP_TO_ERROR(); - } - } - } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; - break; - } + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ /* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4402787d96f12e..1a991608385405 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4401,18 +4401,35 @@ goto error; } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (v == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } goto error; } + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto pop_1_error; if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto pop_1_error; + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto pop_1_error; + } } } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 1b76f1480b4f11..b3787345ec6714 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -740,12 +740,7 @@ break; } - case _LOAD_FROM_DICT_OR_GLOBALS: { - _Py_UopsSymbol *v; - v = sym_new_not_null(ctx); - stack_pointer[-1] = v; - break; - } + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ /* _LOAD_NAME is not a viable micro-op for tier 2 */ From d28afd3fa064db10a2eb2a65bba33e8ea77a8fcf Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 31 May 2024 14:05:51 -0700 Subject: [PATCH 309/903] gh-119180: Lazily wrap annotations on classmethod and staticmethod (#119864) --- Lib/test/test_descr.py | 38 ++++++- ...-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst | 3 + Objects/funcobject.c | 100 +++++++++++++++++- 3 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index c3f292467a6738..7742f075285602 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1593,8 +1593,7 @@ def f(cls, arg): self.fail("classmethod shouldn't accept keyword args") cm = classmethod(f) - cm_dict = {'__annotations__': {}, - '__doc__': ( + cm_dict = {'__doc__': ( "f docstring" if support.HAVE_DOCSTRINGS else None @@ -1610,6 +1609,41 @@ def f(cls, arg): del cm.x self.assertNotHasAttr(cm, "x") + def test_classmethod_staticmethod_annotations(self): + for deco in (classmethod, staticmethod): + @deco + def unannotated(cls): pass + @deco + def annotated(cls) -> int: pass + + for method in (annotated, unannotated): + with self.subTest(deco=deco, method=method): + original_annotations = dict(method.__wrapped__.__annotations__) + self.assertNotIn('__annotations__', method.__dict__) + self.assertEqual(method.__annotations__, original_annotations) + self.assertIn('__annotations__', method.__dict__) + + new_annotations = {"a": "b"} + method.__annotations__ = new_annotations + self.assertEqual(method.__annotations__, new_annotations) + self.assertEqual(method.__wrapped__.__annotations__, original_annotations) + + del method.__annotations__ + self.assertEqual(method.__annotations__, original_annotations) + + original_annotate = method.__wrapped__.__annotate__ + self.assertNotIn('__annotate__', method.__dict__) + self.assertIs(method.__annotate__, original_annotate) + self.assertIn('__annotate__', method.__dict__) + + new_annotate = lambda: {"annotations": 1} + method.__annotate__ = new_annotate + self.assertIs(method.__annotate__, new_annotate) + self.assertIs(method.__wrapped__.__annotate__, original_annotate) + + del method.__annotate__ + self.assertIs(method.__annotate__, original_annotate) + @support.refcount_test def test_refleaks_in_classmethod___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst new file mode 100644 index 00000000000000..1e5ad7d08eed7c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst @@ -0,0 +1,3 @@ +:func:`classmethod` and :func:`staticmethod` now wrap the +:attr:`__annotations__` and :attr:`!__annotate__` attributes of their +underlying callable lazily. See :pep:`649`. Patch by Jelle Zijlstra. diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 4e78252052932c..40211297be20c0 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1172,12 +1172,57 @@ functools_wraps(PyObject *wrapper, PyObject *wrapped) COPY_ATTR(__name__); COPY_ATTR(__qualname__); COPY_ATTR(__doc__); - COPY_ATTR(__annotations__); return 0; #undef COPY_ATTR } +// Used for wrapping __annotations__ and __annotate__ on classmethod +// and staticmethod objects. +static PyObject * +descriptor_get_wrapped_attribute(PyObject *wrapped, PyObject *dict, PyObject *name) +{ + PyObject *res; + if (PyDict_GetItemRef(dict, name, &res) < 0) { + return NULL; + } + if (res != NULL) { + return res; + } + res = PyObject_GetAttr(wrapped, name); + if (res == NULL) { + return NULL; + } + if (PyDict_SetItem(dict, name, res) < 0) { + Py_DECREF(res); + return NULL; + } + return res; +} + +static int +descriptor_set_wrapped_attribute(PyObject *dict, PyObject *name, PyObject *value, + char *type_name) +{ + if (value == NULL) { + if (PyDict_DelItem(dict, name) < 0) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + PyErr_Format(PyExc_AttributeError, + "'%.200s' object has no attribute '%U'", + type_name, name); + } + else { + return -1; + } + } + return 0; + } + else { + return PyDict_SetItem(dict, name, value); + } +} + /* Class method object */ @@ -1283,10 +1328,37 @@ cm_get___isabstractmethod__(classmethod *cm, void *closure) Py_RETURN_FALSE; } +static PyObject * +cm_get___annotations__(classmethod *cm, void *closure) +{ + return descriptor_get_wrapped_attribute(cm->cm_callable, cm->cm_dict, &_Py_ID(__annotations__)); +} + +static int +cm_set___annotations__(classmethod *cm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(cm->cm_dict, &_Py_ID(__annotations__), value, "classmethod"); +} + +static PyObject * +cm_get___annotate__(classmethod *cm, void *closure) +{ + return descriptor_get_wrapped_attribute(cm->cm_callable, cm->cm_dict, &_Py_ID(__annotate__)); +} + +static int +cm_set___annotate__(classmethod *cm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(cm->cm_dict, &_Py_ID(__annotate__), value, "classmethod"); +} + + static PyGetSetDef cm_getsetlist[] = { {"__isabstractmethod__", (getter)cm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__annotations__", (getter)cm_get___annotations__, (setter)cm_set___annotations__, NULL, NULL}, + {"__annotate__", (getter)cm_get___annotate__, (setter)cm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1479,10 +1551,36 @@ sm_get___isabstractmethod__(staticmethod *sm, void *closure) Py_RETURN_FALSE; } +static PyObject * +sm_get___annotations__(staticmethod *sm, void *closure) +{ + return descriptor_get_wrapped_attribute(sm->sm_callable, sm->sm_dict, &_Py_ID(__annotations__)); +} + +static int +sm_set___annotations__(staticmethod *sm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(sm->sm_dict, &_Py_ID(__annotations__), value, "staticmethod"); +} + +static PyObject * +sm_get___annotate__(staticmethod *sm, void *closure) +{ + return descriptor_get_wrapped_attribute(sm->sm_callable, sm->sm_dict, &_Py_ID(__annotate__)); +} + +static int +sm_set___annotate__(staticmethod *sm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(sm->sm_dict, &_Py_ID(__annotate__), value, "staticmethod"); +} + static PyGetSetDef sm_getsetlist[] = { {"__isabstractmethod__", (getter)sm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__annotations__", (getter)sm_get___annotations__, (setter)sm_set___annotations__, NULL, NULL}, + {"__annotate__", (getter)sm_get___annotate__, (setter)sm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; From cc5cd4d93e3e079e897da9ceb1732ef16d79d01b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 31 May 2024 17:08:55 -0500 Subject: [PATCH 310/903] statistics.fmean(): speed-up code path for non-sizeable inputs. (gh-119876) --- Lib/statistics.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index c2f4fe8e054d3d..450edfaabe8def 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -505,13 +505,11 @@ def fmean(data, weights=None): n = len(data) except TypeError: # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - data = count(data) - total = fsum(data) + counter = count() + total = fsum(map(itemgetter(0), zip(data, counter))) + n = next(counter) + else: + total = fsum(data) if not n: raise StatisticsError('fmean requires at least one data point') return total / n From 3859e09e3d92d004978dd838f0511364e7edfb94 Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Sat, 1 Jun 2024 13:59:35 +1000 Subject: [PATCH 311/903] gh-74929: PEP 667 C API documentation (gh-119379) * Add docs for new APIs * Add soft-deprecation notices * Add What's New porting entries * Update comments referencing `PyFrame_LocalsToFast()` to mention the proxy instead * Other related cleanups found when looking for refs to the deprecated APIs --- Doc/c-api/reflection.rst | 38 ++++++++++++++++++++++++++++++++++++++ Doc/data/refcounts.dat | 32 ++++++++++++++++++++++++++++++++ Doc/whatsnew/3.13.rst | 16 +++++++++++++++- Lib/test/test_sys.py | 13 +++++++++---- Objects/frameobject.c | 9 +++++++-- Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 2 +- Python/sysmodule.c | 1 - 9 files changed, 104 insertions(+), 11 deletions(-) diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 4b1c4770848a30..5dcfe40c2ce92b 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -7,18 +7,30 @@ Reflection .. c:function:: PyObject* PyEval_GetBuiltins(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameBuiltins` instead. + Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is currently executing. .. c:function:: PyObject* PyEval_GetLocals(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameLocals` instead. + Return a dictionary of the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. .. c:function:: PyObject* PyEval_GetGlobals(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameGlobals` instead. + Return a dictionary of the global variables in the current execution frame, or ``NULL`` if no frame is currently executing. @@ -31,6 +43,32 @@ Reflection See also :c:func:`PyThreadState_GetFrame`. +.. c:function:: PyObject* PyEval_GetFrameBuiltins(void) + + Return a dictionary of the builtins in the current execution frame, + or the interpreter of the thread state if no frame is currently executing. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyEval_GetFrameLocals(void) + + Return a dictionary of the local variables in the current execution frame, + or ``NULL`` if no frame is currently executing. Equivalent to calling + :func:`locals` in Python code. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyEval_GetFrameGlobals(void) + + Return a dictionary of the global variables in the current execution frame, + or ``NULL`` if no frame is currently executing. Equivalent to calling + :func:`globals` in Python code. + + .. versionadded:: 3.13 + + .. c:function:: const char* PyEval_GetFuncName(PyObject *func) Return the name of *func* if it is a function, class or instance object, else the diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 62a96146d605ff..a7d06e076a1b55 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -790,6 +790,12 @@ PyEval_GetGlobals:PyObject*::0: PyEval_GetFrame:PyObject*::0: +PyEval_GetFrameBuiltins:PyObject*::+1: + +PyEval_GetFrameLocals:PyObject*::+1: + +PyEval_GetFrameGlobals:PyObject*::+1: + PyEval_GetFuncDesc:const char*::: PyEval_GetFuncDesc:PyObject*:func:0: @@ -916,6 +922,32 @@ PyFloat_FromString:PyObject*:str:0: PyFloat_GetInfo:PyObject*::+1: PyFloat_GetInfo::void:: +PyFrame_GetBack:PyObject*::+1: +PyFrame_GetBack:PyFrameObject*:frame:0: + +PyFrame_GetBuiltins:PyObject*::+1: +PyFrame_GetBuiltins:PyFrameObject*:frame:0: + +PyFrame_GetCode:PyObject*::+1: +PyFrame_GetCode:PyFrameObject*:frame:0: + +PyFrame_GetGenerator:PyObject*::+1: +PyFrame_GetGenerator:PyFrameObject*:frame:0: + +PyFrame_GetGlobals:PyObject*::+1: +PyFrame_GetGlobals:PyFrameObject*:frame:0: + +PyFrame_GetLocals:PyObject*::+1: +PyFrame_GetLocals:PyFrameObject*:frame:0: + +PyFrame_GetVar:PyObject*::+1: +PyFrame_GetVar:PyFrameObject*:frame:0: +PyFrame_GetVar:PyObject*:name:0: + +PyFrame_GetVarString:PyObject*::+1: +PyFrame_GetVarString:PyFrameObject*:frame:0: +PyFrame_GetVarString:const char*:name:: + PyFrozenSet_Check:int::: PyFrozenSet_Check:PyObject*:p:0: diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 29bb3b81f6323c..3a52baf71310a3 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -97,7 +97,7 @@ Interpreter improvements: * :pep:`667`: The :func:`locals` builtin now has :ref:`defined semantics ` when mutating the returned mapping. Python debuggers and similar tools may now more reliably - update local variables in optimized frames even during concurrent code + update local variables in optimized scopes even during concurrent code execution. New typing features: @@ -2143,6 +2143,11 @@ New Features destruction the same way the :mod:`tracemalloc` module does. (Contributed by Pablo Galindo in :gh:`93502`.) +* Add :c:func:`PyEval_GetFrameBuiltins`, :c:func:`PyEval_GetFrameGlobals`, and + :c:func:`PyEval_GetFrameLocals` to the C API. These replacements for + :c:func:`PyEval_GetBuiltins`, :c:func:`PyEval_GetGlobals`, and + :c:func:`PyEval_GetLocals` return :term:`strong references ` + rather than borrowed references. (Added as part of :pep:`667`.) Build Changes ============= @@ -2318,6 +2323,15 @@ Changes in the C API to :c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in :gh:`115781`.) +* :c:func:`!PyFrame_FastToLocals` and :c:func:`!PyFrame_FastToLocalsWithError` + no longer have any effect. Calling these functions has been redundant since + Python 3.11, when :c:func:`PyFrame_GetLocals` was first introduced. + (Changed as part of :pep:`667`.) + +* :c:func:`!PyFrame_LocalsToFast` no longer has any effect. Calling this function + is redundant now that :c:func:`PyFrame_GetLocals` returns a write-through proxy + for :term:`optimized scopes `. (Changed as part of :pep:`667`.) + Removed C APIs -------------- diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 8fe1d77756866a..1e5823f8883957 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -394,10 +394,15 @@ def test_dlopenflags(self): @test.support.refcount_test def test_refcount(self): - # n here must be a global in order for this test to pass while - # tracing with a python function. Tracing calls PyFrame_FastToLocals - # which will add a copy of any locals to the frame object, causing - # the reference count to increase by 2 instead of 1. + # n here originally had to be a global in order for this test to pass + # while tracing with a python function. Tracing used to call + # PyFrame_FastToLocals, which would add a copy of any locals to the + # frame object, causing the ref count to increase by 2 instead of 1. + # While that no longer happens (due to PEP 667), this test case retains + # its original global-based implementation + # PEP 683's immortal objects also made this point moot, since the + # refcount for None doesn't change anyway. Maybe this test should be + # using a different constant value? (e.g. an integer) global n self.assertRaises(TypeError, sys.getrefcount) c = sys.getrefcount(None) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index fc8d6c7a7aee89..5c65007dae46d2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1888,8 +1888,7 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, } // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), // with the initial value set when the frame was created... - // (unlikely) ...or it was set to some initial value by - // an earlier call to PyFrame_LocalsToFast(). + // (unlikely) ...or it was set via the f_locals proxy. } } } @@ -2002,18 +2001,24 @@ PyFrame_GetVarString(PyFrameObject *frame, const char *name) int PyFrame_FastToLocalsWithError(PyFrameObject *f) { + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. return 0; } void PyFrame_FastToLocals(PyFrameObject *f) { + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. return; } void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. return; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1c12e1cddbbc10..413ad1105f9428 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1570,7 +1570,7 @@ dummy_func( inst(MAKE_CELL, (--)) { // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). + // via the f_locals proxy before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0dfe490cb37047..bab629684c53f6 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1541,7 +1541,7 @@ case _MAKE_CELL: { oparg = CURRENT_OPARG(); // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). + // via the f_locals proxy before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1a991608385405..355be966cbb84a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4784,7 +4784,7 @@ next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). + // via the f_locals proxy before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 4da13e4552e786..00aa95531026b5 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -35,7 +35,6 @@ Data members: #include "pycore_sysmodule.h" // export _PySys_GetSizeOf() #include "pycore_tuple.h" // _PyTuple_FromArray() -#include "frameobject.h" // PyFrame_FastToLocalsWithError() #include "pydtrace.h" // PyDTrace_AUDIT() #include "osdefs.h" // DELIM #include "stdlib_module_names.h" // _Py_stdlib_module_names From 2180991ea3d50f56595edae241cc92dd4e7de642 Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Sat, 1 Jun 2024 16:21:48 +1000 Subject: [PATCH 312/903] gh-118888: Further PEP 667 docs updates (gh-119893) * Clarify impact on default behaviour of exec, eval, etc * Update documentation for changes to PyEval_GetLocals (gh-74929) Closes gh-11888 --- Doc/c-api/reflection.rst | 21 +++++++++++++++++++-- Doc/whatsnew/3.13.rst | 26 +++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 5dcfe40c2ce92b..af9a1a74ec137e 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -19,11 +19,24 @@ Reflection .. deprecated:: 3.13 - Use :c:func:`PyEval_GetFrameLocals` instead. + To avoid creating a reference cycle in :term:`optimized scopes `, + use either :c:func:`PyEval_GetFrameLocals` to obtain the same behaviour as calling + :func:`locals` in Python code, or else call :c:func:`PyFrame_GetLocals` on the result + of :c:func:`PyEval_GetFrame` to get the same result as this function without having to + cache the proxy instance on the underlying frame. - Return a dictionary of the local variables in the current execution frame, + Return the :attr:`~frame.f_locals` attribute of the currently executing frame, or ``NULL`` if no frame is currently executing. + If the frame refers to an :term:`optimized scope`, this returns a + write-through proxy object that allows modifying the locals. + In all other cases (classes, modules, :func:`exec`, :func:`eval`) it returns + the mapping representing the frame locals directly (as described for + :func:`locals`). + + .. versionchanged:: 3.13 + As part of :pep:`667`, return a proxy object for optimized scopes. + .. c:function:: PyObject* PyEval_GetGlobals(void) @@ -57,6 +70,10 @@ Reflection or ``NULL`` if no frame is currently executing. Equivalent to calling :func:`locals` in Python code. + To access :attr:`~frame.f_locals` on the current frame without making an independent + snapshot in :term:`optimized scopes `, call :c:func:`PyFrame_GetLocals` + on the result of :c:func:`PyEval_GetFrame`. + .. versionadded:: 3.13 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 3a52baf71310a3..ab260bf2a2d740 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -266,6 +266,21 @@ comprehensions, and generator expressions) to explicitly return independent snapshots of the currently assigned local variables, including locally referenced nonlocal variables captured in closures. +This change to the semantics of :func:`locals` in optimized scopes also affects the default +behaviour of code execution functions that implicitly target ``locals()`` if no explicit +namespace is provided (such as :func:`exec` and :func:`eval`). In previous versions, whether +or not changes could be accessed by calling ``locals()`` after calling the code execution +function was implementation dependent. In CPython specifically, such code would typically +appear to work as desired, but could sometimes fail in optimized scopes based on other code +(including debuggers and code execution tracing tools) potentially resetting the shared +snapshot in that scope. Now, the code will always run against an independent snapshot of the +local variables in optimized scopes, and hence the changes will never be visible in +subsequent calls to ``locals()``. To access the changes made in these cases, an explicit +namespace reference must now be passed to the relevant function. Alternatively, it may make +sense to update affected code to use a higher level code execution API that returns the +resulting code execution namespace (e.g. :func:`runpy.run_path` when executing Python +files from disk). + To ensure debuggers and similar tools can reliably update local variables in scopes affected by this change, :attr:`FrameType.f_locals ` now returns a write-through proxy to the frame's local and locally referenced @@ -2235,7 +2250,10 @@ Changes in the Python API independent snapshot on each call, and hence no longer implicitly updates previously returned references. Obtaining the legacy CPython behaviour now requires explicit calls to update the initially returned dictionary with the - results of subsequent calls to ``locals()``. (Changed as part of :pep:`667`.) + results of subsequent calls to ``locals()``. Code execution functions that + implicitly target ``locals()`` (such as ``exec`` and ``eval``) must be + passed an explicit namespace to access their results in an optimized scope. + (Changed as part of :pep:`667`.) * Calling :func:`locals` from a comprehension at module or class scope (including via ``exec`` or ``eval``) once more behaves as if the comprehension @@ -2323,6 +2341,12 @@ Changes in the C API to :c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in :gh:`115781`.) +* Calling :c:func:`PyFrame_GetLocals` or :c:func:`PyEval_GetLocals` in an + :term:`optimized scope` now returns a write-through proxy rather than a + snapshot that gets updated at ill-specified times. If a snapshot is desired, + it must be created explicitly (e.g. with :c:func:`PyDict_Copy`) or by calling + the new :c:func:`PyEval_GetFrameLocals` API. (Changed as part of :pep:`667`.) + * :c:func:`!PyFrame_FastToLocals` and :c:func:`!PyFrame_FastToLocalsWithError` no longer have any effect. Calling these functions has been redundant since Python 3.11, when :c:func:`PyFrame_GetLocals` was first introduced. From 51191dbfdd115b6de481a1cb5c86b952d622c2d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jun 2024 10:11:53 +0000 Subject: [PATCH 313/903] build(deps-dev): bump types-setuptools from 69.5.0.20240423 to 70.0.0.20240524 in /Tools (#119899) --- Tools/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 1767727373918f..f430ddedb9f856 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -4,4 +4,4 @@ mypy==1.10.0 # needed for peg_generator: types-psutil==5.9.5.20240423 -types-setuptools==69.5.0.20240423 +types-setuptools==70.0.0.20240524 From 5152120ae746516670c77e7feed5c4a8912f2bbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jun 2024 10:38:13 +0000 Subject: [PATCH 314/903] Bump types-psutil from 5.9.5.20240423 to 5.9.5.20240516 in /Tools (#119900) --- Tools/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index f430ddedb9f856..44316e3d7d8ac5 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -3,5 +3,5 @@ mypy==1.10.0 # needed for peg_generator: -types-psutil==5.9.5.20240423 +types-psutil==5.9.5.20240516 types-setuptools==70.0.0.20240524 From 60593b2052ca275559c11028d50e19f8e5dfee13 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Sat, 1 Jun 2024 10:04:05 -0400 Subject: [PATCH 315/903] gh-117657: Fix TSAN race in free-threaded GC (#119883) Only call `gc_restore_tid()` from stop-the-world contexts. `worklist_pop()` can be called while other threads are running, so use a relaxed atomic to modify `ob_tid`. --- Python/gc_free_threading.c | 5 ++--- Tools/tsan/suppressions_free_threading.txt | 3 --- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index ee006bb4aa12b7..e6bd012c40ee82 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -86,7 +86,7 @@ worklist_pop(struct worklist *worklist) PyObject *op = (PyObject *)worklist->head; if (op != NULL) { worklist->head = op->ob_tid; - op->ob_tid = 0; + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); } return op; } @@ -189,6 +189,7 @@ merge_refcount(PyObject *op, Py_ssize_t extra) static void gc_restore_tid(PyObject *op) { + assert(_PyInterpreterState_GET()->stoptheworld.world_stopped); mi_segment_t *segment = _mi_ptr_segment(op); if (_Py_REF_IS_MERGED(op->ob_ref_shared)) { op->ob_tid = 0; @@ -676,7 +677,6 @@ call_weakref_callbacks(struct collection_state *state) Py_DECREF(temp); } - gc_restore_tid(op); Py_DECREF(op); // drop worklist reference } } @@ -986,7 +986,6 @@ cleanup_worklist(struct worklist *worklist) { PyObject *op; while ((op = worklist_pop(worklist)) != NULL) { - gc_restore_tid(op); gc_clear_unreachable(op); Py_DECREF(op); } diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index cda57d78067bb3..951635e7c6533d 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -37,7 +37,6 @@ race_top:_PyImport_ReleaseLock race_top:_PyParkingLot_Park race_top:_PyType_HasFeature race_top:assign_version_tag -race_top:gc_restore_tid race_top:insertdict race_top:lookup_tp_dict race_top:mi_heap_visit_pages @@ -64,7 +63,6 @@ race_top:list_get_item_ref race_top:make_pending_calls race_top:set_add_entry race_top:should_intern_string -race_top:worklist_pop race_top:_PyEval_IsGILEnabled race_top:llist_insert_tail race_top:_Py_slot_tp_getattr_hook @@ -86,7 +84,6 @@ race_top:sock_close race_top:tstate_delete_common race_top:tstate_is_freed race_top:type_modified_unlocked -race_top:update_refs race_top:write_thread_id race_top:PyThreadState_Clear From 90ec19fd33e2452902b9788d4821f1fbf6542304 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Sat, 1 Jun 2024 10:04:38 -0400 Subject: [PATCH 316/903] gh-117657: Fix TSAN race in QSBR assertion (#119887) Due to a limitation in TSAN, all reads from `PyThreadState.state` must be atomic to avoid reported races. --- Python/qsbr.c | 3 ++- Tools/tsan/suppressions_free_threading.txt | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Python/qsbr.c b/Python/qsbr.c index 9cbce9044e2941..a7321154a62ffc 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -160,7 +160,8 @@ qsbr_poll_scan(struct _qsbr_shared *shared) bool _Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) { - assert(_PyThreadState_GET()->state == _Py_THREAD_ATTACHED); + assert(_Py_atomic_load_int_relaxed(&_PyThreadState_GET()->state) == _Py_THREAD_ATTACHED); + if (_Py_qbsr_goal_reached(qsbr, goal)) { return true; } diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 951635e7c6533d..9a53990f8b2ff8 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -75,8 +75,6 @@ race_top:_PyFrame_GetCode race_top:_PyFrame_Initialize race_top:PyInterpreterState_ThreadHead race_top:_PyObject_TryGetInstanceAttribute -race_top:_Py_qsbr_unregister -race_top:_Py_qsbr_poll race_top:PyThreadState_Next race_top:Py_TYPE race_top:PyUnstable_InterpreterFrame_GetLine From ce2ea7d629788fd051cbec099b5947ecbe50e819 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Jun 2024 10:49:14 -0500 Subject: [PATCH 317/903] Minor speed/accuracy improvement for kde() (gh-119910) --- Lib/statistics.py | 17 +++++++++-------- Lib/test/test_statistics.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index 450edfaabe8def..c36145fe7f2a79 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -953,12 +953,14 @@ def kde(data, h, kernel='normal', *, cumulative=False): case 'quartic' | 'biweight': K = lambda t: 15/16 * (1.0 - t * t) ** 2 - W = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2 + W = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), + (t**5, t**3, t, 1.0)) support = 1.0 case 'triweight': K = lambda t: 35/32 * (1.0 - t * t) ** 3 - W = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2 + W = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), + (t**7, t**5, t**3, t, 1.0)) support = 1.0 case 'cosine': @@ -974,12 +976,10 @@ def kde(data, h, kernel='normal', *, cumulative=False): if support is None: def pdf(x): - n = len(data) - return sum(K((x - x_i) / h) for x_i in data) / (n * h) + return sum(K((x - x_i) / h) for x_i in data) / (len(data) * h) def cdf(x): - n = len(data) - return sum(W((x - x_i) / h) for x_i in data) / n + return sum(W((x - x_i) / h) for x_i in data) / len(data) else: @@ -1732,7 +1732,7 @@ def _quartic_invcdf_estimate(p): _quartic_invcdf = _newton_raphson( f_inv_estimate = _quartic_invcdf_estimate, - f = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2, + f = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), (t**5, t**3, t, 1.0)), f_prime = lambda t: 15/16 * (1.0 - t * t) ** 2) def _triweight_invcdf_estimate(p): @@ -1742,7 +1742,8 @@ def _triweight_invcdf_estimate(p): _triweight_invcdf = _newton_raphson( f_inv_estimate = _triweight_invcdf_estimate, - f = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2, + f = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), + (t**7, t**5, t**3, t, 1.0)), f_prime = lambda t: 35/32 * (1.0 - t * t) ** 3) _kernel_invcdfs = { diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 6f68edd447c953..cded8aba6e8cd7 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2444,7 +2444,7 @@ def test_kde_kernel_invcdfs(self): with self.subTest(kernel=kernel): cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) for x in xarr: - self.assertAlmostEqual(invcdf(cdf(x)), x, places=5) + self.assertAlmostEqual(invcdf(cdf(x)), x, places=6) @support.requires_resource('cpu') def test_kde_random(self): From cf3bba3f0671d2c9fee099e3ab0f78b98b176131 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 1 Jun 2024 19:05:19 +0300 Subject: [PATCH 318/903] gh-113892: Add a extra check to `ProactorEventLoop.sock_connect` to ensure that the given socket is in non-blocking mode (#119519) --- Lib/asyncio/proactor_events.py | 2 ++ Lib/test/test_asyncio/test_proactor_events.py | 9 +++++++-- .../2024-05-24-21-54-55.gh-issue-113892.JKDFqq.rst | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-24-21-54-55.gh-issue-113892.JKDFqq.rst diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 397a8cda757895..7eb55bd63ddb73 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -721,6 +721,8 @@ async def sock_sendto(self, sock, data, address): return await self._proactor.sendto(sock, data, 0, address) async def sock_connect(self, sock, address): + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") return await self._proactor.connect(sock, address) async def sock_accept(self, sock): diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index fcaa2f6ade2b76..4b3d551dd7b3a2 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -1018,9 +1018,9 @@ def setUp(self): self.addCleanup(self.file.close) super().setUp() - def make_socket(self, cleanup=True): + def make_socket(self, cleanup=True, blocking=False): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setblocking(False) + sock.setblocking(blocking) sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024) if cleanup: @@ -1082,6 +1082,11 @@ def test_sock_sendfile_not_regular_file(self): 0, None)) self.assertEqual(self.file.tell(), 0) + def test_blocking_socket(self): + self.loop.set_debug(True) + sock = self.make_socket(blocking=True) + with self.assertRaisesRegex(ValueError, "must be non-blocking"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2024-05-24-21-54-55.gh-issue-113892.JKDFqq.rst b/Misc/NEWS.d/next/Library/2024-05-24-21-54-55.gh-issue-113892.JKDFqq.rst new file mode 100644 index 00000000000000..639d5abe878344 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-24-21-54-55.gh-issue-113892.JKDFqq.rst @@ -0,0 +1,3 @@ +Now, the method ``sock_connect`` of :class:`asyncio.ProactorEventLoop` +raises a :exc:`ValueError` if given socket is not in +non-blocking mode, as well as in other loop implementations. From 7dc745d1f5d9558047a52cad5e01df7567533269 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Sat, 1 Jun 2024 12:15:58 -0400 Subject: [PATCH 319/903] gh-117657: Add TSAN suppression for `set_discard_entry` (#119908) Seen in CI occasionally when running `test_weakref`. --- Tools/tsan/suppressions_free_threading.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 9a53990f8b2ff8..f855e9ce2698a5 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -44,6 +44,8 @@ race_top:PyMember_GetOne race_top:PyMember_SetOne race_top:new_reference race_top:set_contains_key +# https://gist.github.com/colesbury/d13d033f413b4ad07929d044bed86c35 +race_top:set_discard_entry race_top:set_inheritable race_top:start_the_world race_top:tstate_set_detached From 63111bfcf021db29ce6ef9ffa4117ffb7a2cb868 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Jun 2024 11:30:24 -0500 Subject: [PATCH 320/903] Add unique() recipe to itertools docs (gh-119911) --- Doc/library/itertools.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 121bfd3de343c4..3dc3f60923a0ba 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -857,7 +857,7 @@ and :term:`generators ` which incur interpreter overhead. return len(take(2, groupby(iterable, key))) <= 1 def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." + "Yield unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') → A B C D A B # unique_justseen('ABBcCAD', str.casefold) → A B c A D if key is None: @@ -865,7 +865,7 @@ and :term:`generators ` which incur interpreter overhead. return map(next, map(operator.itemgetter(1), groupby(iterable, key))) def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." + "Yield unique elements, preserving order. Remember all elements ever seen." # unique_everseen('AAAABBBCCDAABBB') → A B C D # unique_everseen('ABBcCAD', str.casefold) → A B c D seen = set() @@ -880,6 +880,11 @@ and :term:`generators ` which incur interpreter overhead. seen.add(k) yield element + def unique(iterable, key=None, reverse=False): + "Yield unique elements in sorted order. Supports unhashable inputs." + # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] + return unique_justseen(sorted(iterable, key=key, reverse=reverse), key=key) + def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG @@ -1605,6 +1610,13 @@ The following recipes have a more mathematical flavor: >>> ''.join(input_iterator) 'AAABBBCCDAABBB' + >>> list(unique([[1, 2], [3, 4], [1, 2]])) + [[1, 2], [3, 4]] + >>> list(unique('ABBcCAD', str.casefold)) + ['A', 'B', 'c', 'D'] + >>> list(unique('ABBcCAD', str.casefold, reverse=True)) + ['D', 'c', 'B', 'A'] + >>> d = dict(a=1, b=2, c=3) >>> it = iter_except(d.popitem, KeyError) >>> d['d'] = 4 From 53b1981fb0cda6c656069e992f172fc6aad7c99c Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 1 Jun 2024 19:49:12 +0100 Subject: [PATCH 321/903] GH-89727: Fix `shutil.rmtree()` recursion error on deep trees (#119808) Implement `shutil._rmtree_safe_fd()` using a list as a stack to avoid emitting recursion errors on deeply nested trees. `shutil._rmtree_unsafe()` was fixed in a150679f90. --- Lib/shutil.py | 162 +++++++----------- Lib/test/test_shutil.py | 1 - ...4-05-30-21-37-05.gh-issue-89727.D6S9ig.rst | 2 + 3 files changed, 68 insertions(+), 97 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-30-21-37-05.gh-issue-89727.D6S9ig.rst diff --git a/Lib/shutil.py b/Lib/shutil.py index 03a9d756030430..b0d49e98cfe5f9 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -635,81 +635,76 @@ def onerror(err): onexc(os.rmdir, path, err) # Version using fd-based APIs to protect against races -def _rmtree_safe_fd(topfd, path, onexc): +def _rmtree_safe_fd(stack, onexc): + # Each stack item has four elements: + # * func: The first operation to perform: os.lstat, os.close or os.rmdir. + # Walking a directory starts with an os.lstat() to detect symlinks; in + # this case, func is updated before subsequent operations and passed to + # onexc() if an error occurs. + # * dirfd: Open file descriptor, or None if we're processing the top-level + # directory given to rmtree() and the user didn't supply dir_fd. + # * path: Path of file to operate upon. This is passed to onexc() if an + # error occurs. + # * orig_entry: os.DirEntry, or None if we're processing the top-level + # directory given to rmtree(). We used the cached stat() of the entry to + # save a call to os.lstat() when walking subdirectories. + func, dirfd, path, orig_entry = stack.pop() + name = path if orig_entry is None else orig_entry.name try: + if func is os.close: + os.close(dirfd) + return + if func is os.rmdir: + os.rmdir(name, dir_fd=dirfd) + return + + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + assert func is os.lstat + if orig_entry is None: + orig_st = os.lstat(name, dir_fd=dirfd) + else: + orig_st = orig_entry.stat(follow_symlinks=False) + + func = os.open # For error reporting. + topfd = os.open(name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dirfd) + + func = os.path.islink # For error reporting. + try: + if not os.path.samestat(orig_st, os.fstat(topfd)): + # Symlinks to directories are forbidden, see GH-46010. + raise OSError("Cannot call rmtree on a symbolic link") + stack.append((os.rmdir, dirfd, path, orig_entry)) + finally: + stack.append((os.close, topfd, path, orig_entry)) + + func = os.scandir # For error reporting. with os.scandir(topfd) as scandir_it: entries = list(scandir_it) - except FileNotFoundError: - return - except OSError as err: - err.filename = path - onexc(os.scandir, path, err) - return - for entry in entries: - fullname = os.path.join(path, entry.name) - try: - is_dir = entry.is_dir(follow_symlinks=False) - except FileNotFoundError: - continue - except OSError: - is_dir = False - else: - if is_dir: - try: - orig_st = entry.stat(follow_symlinks=False) - is_dir = stat.S_ISDIR(orig_st.st_mode) - except FileNotFoundError: - continue - except OSError as err: - onexc(os.lstat, fullname, err) - continue - if is_dir: + for entry in entries: + fullname = os.path.join(path, entry.name) try: - dirfd = os.open(entry.name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=topfd) - dirfd_closed = False + if entry.is_dir(follow_symlinks=False): + # Traverse into sub-directory. + stack.append((os.lstat, topfd, fullname, entry)) + continue except FileNotFoundError: continue - except OSError as err: - onexc(os.open, fullname, err) - else: - try: - if os.path.samestat(orig_st, os.fstat(dirfd)): - _rmtree_safe_fd(dirfd, fullname, onexc) - try: - os.close(dirfd) - except OSError as err: - # close() should not be retried after an error. - dirfd_closed = True - onexc(os.close, fullname, err) - dirfd_closed = True - try: - os.rmdir(entry.name, dir_fd=topfd) - except FileNotFoundError: - continue - except OSError as err: - onexc(os.rmdir, fullname, err) - else: - try: - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or stat.S_ISDIR above. - raise OSError("Cannot call rmtree on a symbolic " - "link") - except OSError as err: - onexc(os.path.islink, fullname, err) - finally: - if not dirfd_closed: - try: - os.close(dirfd) - except OSError as err: - onexc(os.close, fullname, err) - else: + except OSError: + pass try: os.unlink(entry.name, dir_fd=topfd) except FileNotFoundError: continue except OSError as err: onexc(os.unlink, fullname, err) + except FileNotFoundError as err: + if orig_entry is None or func is os.close: + err.filename = path + onexc(func, path, err) + except OSError as err: + err.filename = path + onexc(func, path, err) _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and @@ -762,41 +757,16 @@ def onexc(*args): # While the unsafe rmtree works fine on bytes, the fd based does not. if isinstance(path, bytes): path = os.fsdecode(path) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - try: - orig_st = os.lstat(path, dir_fd=dir_fd) - except OSError as err: - onexc(os.lstat, path, err) - return + stack = [(os.lstat, dir_fd, path, None)] try: - fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dir_fd) - fd_closed = False - except OSError as err: - onexc(os.open, path, err) - return - try: - if os.path.samestat(orig_st, os.fstat(fd)): - _rmtree_safe_fd(fd, path, onexc) - try: - os.close(fd) - except OSError as err: - # close() should not be retried after an error. - fd_closed = True - onexc(os.close, path, err) - fd_closed = True - try: - os.rmdir(path, dir_fd=dir_fd) - except OSError as err: - onexc(os.rmdir, path, err) - else: - try: - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) + while stack: + _rmtree_safe_fd(stack, onexc) finally: - if not fd_closed: + # Close any file descriptors still on the stack. + while stack: + func, fd, path, entry = stack.pop() + if func is not os.close: + continue try: os.close(fd) except OSError as err: diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 01f139073dcd97..bccb81e0737c57 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -741,7 +741,6 @@ def _onexc(fn, path, exc): shutil.rmtree(TESTFN) raise - @unittest.skipIf(shutil._use_fd_functions, "fd-based functions remain unfixed (GH-89727)") def test_rmtree_above_recursion_limit(self): recursion_limit = 40 # directory_depth > recursion_limit diff --git a/Misc/NEWS.d/next/Library/2024-05-30-21-37-05.gh-issue-89727.D6S9ig.rst b/Misc/NEWS.d/next/Library/2024-05-30-21-37-05.gh-issue-89727.D6S9ig.rst new file mode 100644 index 00000000000000..854c56609acb8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-30-21-37-05.gh-issue-89727.D6S9ig.rst @@ -0,0 +1,2 @@ +Fix issue with :func:`shutil.rmtree` where a :exc:`RecursionError` is raised +on deep directory trees. From c618f7d80e78f83cc24b6bdead33ca38cbd4d27f Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Sat, 1 Jun 2024 23:20:00 +0200 Subject: [PATCH 322/903] gh-119016: Remove outdated sentences from the "classes" tutorial (#119130) Co-authored-by: Alex Waygood --- Doc/tutorial/classes.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 7ab528acb370f2..1b64741c349ee9 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -338,11 +338,7 @@ code will print the value ``16``, without leaving a trace:: del x.counter The other kind of instance attribute reference is a *method*. A method is a -function that "belongs to" an object. (In Python, the term method is not unique -to class instances: other object types can have methods as well. For example, -list objects have methods called append, insert, remove, sort, and so on. -However, in the following discussion, we'll use the term method exclusively to -mean methods of class instance objects, unless explicitly stated otherwise.) +function that "belongs to" an object. .. index:: pair: object; method From e378dc15b52985724b6ae4782c4ef0afc3393ca9 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Jun 2024 22:07:46 -0500 Subject: [PATCH 323/903] Refactor (mostly rearrange) the statistics module (gh-119930) --- Lib/statistics.py | 1851 ++++++++++++++++++----------------- Lib/test/test_statistics.py | 5 +- 2 files changed, 952 insertions(+), 904 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index c36145fe7f2a79..c64c6fae4ab010 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -147,445 +147,148 @@ _SQRT2 = sqrt(2.0) _random = random -# === Exceptions === +## Exceptions ############################################################## class StatisticsError(ValueError): pass -# === Private utilities === +## Measures of central tendency (averages) ################################# -def _sum(data): - """_sum(data) -> (type, sum, count) - - Return a high-precision sum of the given numeric data as a fraction, - together with the type to be converted to and the count of items. - - Examples - -------- - - >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) - (, Fraction(19, 2), 5) - - Some sources of round-off error will be avoided: - - # Built-in sum returns zero. - >>> _sum([1e50, 1, -1e50] * 1000) - (, Fraction(1000, 1), 3000) +def mean(data): + """Return the sample arithmetic mean of data. - Fractions and Decimals are also supported: + >>> mean([1, 2, 3, 4, 4]) + 2.8 >>> from fractions import Fraction as F - >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) - (, Fraction(63, 20), 4) + >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) + Fraction(13, 21) >>> from decimal import Decimal as D - >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] - >>> _sum(data) - (, Fraction(6963, 10000), 4) + >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) + Decimal('0.5625') + + If ``data`` is empty, StatisticsError will be raised. - Mixed types are currently treated as an error, except that int is - allowed. """ - count = 0 - types = set() - types_add = types.add - partials = {} - partials_get = partials.get - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - partials[d] = partials_get(d, 0) + n - if None in partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - total = partials[None] - assert not _isfinite(total) - else: - # Sum all the partial sums using builtin sum. - total = sum(Fraction(n, d) for d, n in partials.items()) - T = reduce(_coerce, types, int) # or raise TypeError - return (T, total, count) + T, total, n = _sum(data) + if n < 1: + raise StatisticsError('mean requires at least one data point') + return _convert(total / n, T) -def _ss(data, c=None): - """Return the exact mean and sum of square deviations of sequence data. +def fmean(data, weights=None): + """Convert data to floats and compute the arithmetic mean. - Calculations are done in a single pass, allowing the input to be an iterator. + This runs faster than the mean() function and it always returns a float. + If the input dataset is empty, it raises a StatisticsError. - If given *c* is used the mean; otherwise, it is calculated from the data. - Use the *c* argument with care, as it can lead to garbage results. + >>> fmean([3.5, 4.0, 5.25]) + 4.25 """ - if c is not None: - T, ssd, count = _sum((d := x - c) * d for x in data) - return (T, ssd, c, count) - count = 0 - types = set() - types_add = types.add - sx_partials = defaultdict(int) - sxx_partials = defaultdict(int) - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - sx_partials[d] += n - sxx_partials[d] += n * n - if not count: - ssd = c = Fraction(0) - elif None in sx_partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - ssd = c = sx_partials[None] - assert not _isfinite(ssd) - else: - sx = sum(Fraction(n, d) for d, n in sx_partials.items()) - sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) - # This formula has poor numeric properties for floats, - # but with fractions it is exact. - ssd = (count * sxx - sx * sx) / count - c = sx / count - T = reduce(_coerce, types, int) # or raise TypeError - return (T, ssd, c, count) + if weights is None: + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + counter = count() + total = fsum(map(itemgetter(0), zip(data, counter))) + n = next(counter) + else: + total = fsum(data) -def _isfinite(x): - try: - return x.is_finite() # Likely a Decimal. - except AttributeError: - return math.isfinite(x) # Coerces to float first. + if not n: + raise StatisticsError('fmean requires at least one data point') + return total / n -def _coerce(T, S): - """Coerce types T and S to a common type, or raise TypeError. + if not isinstance(weights, (list, tuple)): + weights = list(weights) - Coercion rules are currently an implementation detail. See the CoerceTest - test class in test_statistics for details. - """ - # See http://bugs.python.org/issue24068. - assert T is not bool, "initial type T is bool" - # If the types are the same, no need to coerce anything. Put this - # first, so that the usual case (no coercion needed) happens as soon - # as possible. - if T is S: return T - # Mixed int & other coerce to the other type. - if S is int or S is bool: return T - if T is int: return S - # If one is a (strict) subclass of the other, coerce to the subclass. - if issubclass(S, T): return S - if issubclass(T, S): return T - # Ints coerce to the other type. - if issubclass(T, int): return S - if issubclass(S, int): return T - # Mixed fraction & float coerces to float (or float subclass). - if issubclass(T, Fraction) and issubclass(S, float): - return S - if issubclass(T, float) and issubclass(S, Fraction): - return T - # Any other combination is disallowed. - msg = "don't know how to coerce %s and %s" - raise TypeError(msg % (T.__name__, S.__name__)) + try: + num = sumprod(data, weights) + except ValueError: + raise StatisticsError('data and weights must be the same length') + den = fsum(weights) -def _exact_ratio(x): - """Return Real number x to exact (numerator, denominator) pair. + if not den: + raise StatisticsError('sum of weights must be non-zero') - >>> _exact_ratio(0.25) - (1, 4) + return num / den - x is expected to be an int, Fraction, Decimal or float. - """ - # XXX We should revisit whether using fractions to accumulate exact - # ratios is the right way to go. +def geometric_mean(data): + """Convert data to floats and compute the geometric mean. - # The integer ratios for binary floats can have numerators or - # denominators with over 300 decimal digits. The problem is more - # acute with decimal floats where the default decimal context - # supports a huge range of exponents from Emin=-999999 to - # Emax=999999. When expanded with as_integer_ratio(), numbers like - # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large - # numerators or denominators that will slow computation. + Raises a StatisticsError if the input dataset is empty + or if it contains a negative value. - # When the integer ratios are accumulated as fractions, the size - # grows to cover the full range from the smallest magnitude to the - # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300), - # has a 616 digit numerator. Likewise, - # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000')) - # has 10,003 digit numerator. + Returns zero if the product of inputs is zero. - # This doesn't seem to have been problem in practice, but it is a - # potential pitfall. + No special efforts are made to achieve exact results. + (However, this may change in the future.) - try: - return x.as_integer_ratio() - except AttributeError: - pass - except (OverflowError, ValueError): - # float NAN or INF. - assert not _isfinite(x) - return (x, None) - try: - # x may be an Integral ABC. - return (x.numerator, x.denominator) - except AttributeError: - msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" - raise TypeError(msg) + >>> round(geometric_mean([54, 24, 36]), 9) + 36.0 + """ + n = 0 + found_zero = False -def _convert(value, T): - """Convert value to given numeric type T.""" - if type(value) is T: - # This covers the cases where T is Fraction, or where value is - # a NAN or INF (Decimal or float). - return value - if issubclass(T, int) and value.denominator != 1: - T = float - try: - # FIXME: what do we do if this overflows? - return T(value) - except TypeError: - if issubclass(T, Decimal): - return T(value.numerator) / T(value.denominator) - else: - raise + def count_positive(iterable): + nonlocal n, found_zero + for n, x in enumerate(iterable, start=1): + if x > 0.0 or math.isnan(x): + yield x + elif x == 0.0: + found_zero = True + else: + raise StatisticsError('No negative inputs allowed', x) + total = fsum(map(log, count_positive(data))) + if not n: + raise StatisticsError('Must have a non-empty dataset') + if math.isnan(total): + return math.nan + if found_zero: + return math.nan if total == math.inf else 0.0 -def _fail_neg(values, errmsg='negative value'): - """Iterate over values, failing if any are less than zero.""" - for x in values: - if x < 0: - raise StatisticsError(errmsg) - yield x + return exp(total / n) -def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: - """Rank order a dataset. The lowest value has rank 1. +def harmonic_mean(data, weights=None): + """Return the harmonic mean of data. - Ties are averaged so that equal values receive the same rank: + The harmonic mean is the reciprocal of the arithmetic mean of the + reciprocals of the data. It can be used for averaging ratios or + rates, for example speeds. - >>> data = [31, 56, 31, 25, 75, 18] - >>> _rank(data) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + Suppose a car travels 40 km/hr for 5 km and then speeds-up to + 60 km/hr for another 5 km. What is the average speed? - The operation is idempotent: + >>> harmonic_mean([40, 60]) + 48.0 - >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? - It is possible to rank the data in reverse order so that the - highest value has rank 1. Also, a key-function can extract - the field to be ranked: - - >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] - >>> _rank(goals, key=itemgetter(1), reverse=True) - [2.0, 1.0, 3.0] - - Ranks are conventionally numbered starting from one; however, - setting *start* to zero allows the ranks to be used as array indices: - - >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] - >>> scores = [8.1, 7.3, 9.4, 8.3] - >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] - ['Bronze', 'Certificate', 'Gold', 'Silver'] - - """ - # If this function becomes public at some point, more thought - # needs to be given to the signature. A list of ints is - # plausible when ties is "min" or "max". When ties is "average", - # either list[float] or list[Fraction] is plausible. - - # Default handling of ties matches scipy.stats.mstats.spearmanr. - if ties != 'average': - raise ValueError(f'Unknown tie resolution method: {ties!r}') - if key is not None: - data = map(key, data) - val_pos = sorted(zip(data, count()), reverse=reverse) - i = start - 1 - result = [0] * len(val_pos) - for _, g in groupby(val_pos, key=itemgetter(0)): - group = list(g) - size = len(group) - rank = i + (size + 1) / 2 - for value, orig_pos in group: - result[orig_pos] = rank - i += size - return result - - -def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: - """Square root of n/m, rounded to the nearest integer using round-to-odd.""" - # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf - a = math.isqrt(n // m) - return a | (a*a*m != n) - - -# For 53 bit precision floats, the bit width used in -# _float_sqrt_of_frac() is 109. -_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 - - -def _float_sqrt_of_frac(n: int, m: int) -> float: - """Square root of n/m as a float, correctly rounded.""" - # See principle and proof sketch at: https://bugs.python.org/msg407078 - q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 - if q >= 0: - numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q - denominator = 1 - else: - numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) - denominator = 1 << -q - return numerator / denominator # Convert to float - - -def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: - """Square root of n/m as a Decimal, correctly rounded.""" - # Premise: For decimal, computing (n/m).sqrt() can be off - # by 1 ulp from the correctly rounded result. - # Method: Check the result, moving up or down a step if needed. - if n <= 0: - if not n: - return Decimal('0.0') - n, m = -n, -m - - root = (Decimal(n) / Decimal(m)).sqrt() - nr, dr = root.as_integer_ratio() - - plus = root.next_plus() - np, dp = plus.as_integer_ratio() - # test: n / m > ((root + plus) / 2) ** 2 - if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: - return plus - - minus = root.next_minus() - nm, dm = minus.as_integer_ratio() - # test: n / m < ((root + minus) / 2) ** 2 - if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: - return minus - - return root - - -# === Measures of central tendency (averages) === - -def mean(data): - """Return the sample arithmetic mean of data. - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - If ``data`` is empty, StatisticsError will be raised. - """ - T, total, n = _sum(data) - if n < 1: - raise StatisticsError('mean requires at least one data point') - return _convert(total / n, T) - - -def fmean(data, weights=None): - """Convert data to floats and compute the arithmetic mean. - - This runs faster than the mean() function and it always returns a float. - If the input dataset is empty, it raises a StatisticsError. - - >>> fmean([3.5, 4.0, 5.25]) - 4.25 - """ - if weights is None: - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - counter = count() - total = fsum(map(itemgetter(0), zip(data, counter))) - n = next(counter) - else: - total = fsum(data) - if not n: - raise StatisticsError('fmean requires at least one data point') - return total / n - if not isinstance(weights, (list, tuple)): - weights = list(weights) - try: - num = sumprod(data, weights) - except ValueError: - raise StatisticsError('data and weights must be the same length') - den = fsum(weights) - if not den: - raise StatisticsError('sum of weights must be non-zero') - return num / den - - -def geometric_mean(data): - """Convert data to floats and compute the geometric mean. - - Raises a StatisticsError if the input dataset is empty - or if it contains a negative value. - - Returns zero if the product of inputs is zero. - - No special efforts are made to achieve exact results. - (However, this may change in the future.) - - >>> round(geometric_mean([54, 24, 36]), 9) - 36.0 - """ - n = 0 - found_zero = False - def count_positive(iterable): - nonlocal n, found_zero - for n, x in enumerate(iterable, start=1): - if x > 0.0 or math.isnan(x): - yield x - elif x == 0.0: - found_zero = True - else: - raise StatisticsError('No negative inputs allowed', x) - total = fsum(map(log, count_positive(data))) - if not n: - raise StatisticsError('Must have a non-empty dataset') - if math.isnan(total): - return math.nan - if found_zero: - return math.nan if total == math.inf else 0.0 - return exp(total / n) - - -def harmonic_mean(data, weights=None): - """Return the harmonic mean of data. - - The harmonic mean is the reciprocal of the arithmetic mean of the - reciprocals of the data. It can be used for averaging ratios or - rates, for example speeds. - - Suppose a car travels 40 km/hr for 5 km and then speeds-up to - 60 km/hr for another 5 km. What is the average speed? - - >>> harmonic_mean([40, 60]) - 48.0 - - Suppose a car travels 40 km/hr for 5 km, and when traffic clears, - speeds-up to 60 km/hr for the remaining 30 km of the journey. What - is the average speed? - - >>> harmonic_mean([40, 60], weights=[5, 30]) - 56.0 + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 If ``data`` is empty, or any element is less than zero, ``harmonic_mean`` will raise ``StatisticsError``. + """ if iter(data) is data: data = list(data) + errmsg = 'harmonic mean does not support negative values' + n = len(data) if n < 1: raise StatisticsError('harmonic_mean requires at least one data point') @@ -597,6 +300,7 @@ def harmonic_mean(data, weights=None): return x else: raise TypeError('unsupported type') + if weights is None: weights = repeat(1, n) sum_weights = n @@ -606,16 +310,19 @@ def harmonic_mean(data, weights=None): if len(weights) != n: raise StatisticsError('Number of weights does not match data size') _, sum_weights, _ = _sum(w for w in _fail_neg(weights, errmsg)) + try: data = _fail_neg(data, errmsg) T, total, count = _sum(w / x if w else 0 for w, x in zip(weights, data)) except ZeroDivisionError: return 0 + if total <= 0: raise StatisticsError('Weighted sum must be positive') + return _convert(sum_weights / total, T) -# FIXME: investigate ways to calculate medians without sorting? Quickselect? + def median(data): """Return the median (middle value) of numeric data. @@ -652,6 +359,9 @@ def median_low(data): 3 """ + # Potentially the sorting step could be replaced with a quickselect. + # However, it would require an excellent implementation to beat our + # highly optimized builtin sort. data = sorted(data) n = len(data) if n == 0: @@ -795,6 +505,7 @@ def multimode(data): ['b', 'd', 'f'] >>> multimode('') [] + """ counts = Counter(iter(data)) if not counts: @@ -803,337 +514,36 @@ def multimode(data): return [value for value, count in counts.items() if count == maxcount] -def kde(data, h, kernel='normal', *, cumulative=False): - """Kernel Density Estimation: Create a continuous probability density - function or cumulative distribution function from discrete samples. - - The basic idea is to smooth the data using a kernel function - to help draw inferences about a population from a sample. +## Measures of spread ###################################################### - The degree of smoothing is controlled by the scaling parameter h - which is called the bandwidth. Smaller values emphasize local - features while larger values give smoother results. +def variance(data, xbar=None): + """Return the sample variance of data. - The kernel determines the relative weights of the sample data - points. Generally, the choice of kernel shape does not matter - as much as the more influential bandwidth smoothing parameter. + data should be an iterable of Real-valued numbers, with at least two + values. The optional argument xbar, if given, should be the mean of + the data. If it is missing or None, the mean is automatically calculated. - Kernels that give some weight to every sample point: + Use this function when your data is a sample from a population. To + calculate the variance from the entire population, see ``pvariance``. - normal (gauss) - logistic - sigmoid + Examples: - Kernels that only give weight to sample points within - the bandwidth: + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 - rectangular (uniform) - triangular - parabolic (epanechnikov) - quartic (biweight) - triweight - cosine + If you have already calculated the mean of your data, you can pass it as + the optional second argument ``xbar`` to avoid recalculating it: - If *cumulative* is true, will return a cumulative distribution function. + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 - A StatisticsError will be raised if the data sequence is empty. + This function does not check that ``xbar`` is actually the mean of + ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or + impossible results. - Example - ------- - - Given a sample of six data points, construct a continuous - function that estimates the underlying probability density: - - >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> f_hat = kde(sample, h=1.5) - - Compute the area under the curve: - - >>> area = sum(f_hat(x) for x in range(-20, 20)) - >>> round(area, 4) - 1.0 - - Plot the estimated probability density function at - evenly spaced points from -6 to 10: - - >>> for x in range(-6, 11): - ... density = f_hat(x) - ... plot = ' ' * int(density * 400) + 'x' - ... print(f'{x:2}: {density:.3f} {plot}') - ... - -6: 0.002 x - -5: 0.009 x - -4: 0.031 x - -3: 0.070 x - -2: 0.111 x - -1: 0.125 x - 0: 0.110 x - 1: 0.086 x - 2: 0.068 x - 3: 0.059 x - 4: 0.066 x - 5: 0.082 x - 6: 0.082 x - 7: 0.058 x - 8: 0.028 x - 9: 0.009 x - 10: 0.002 x - - Estimate P(4.5 < X <= 7.5), the probability that a new sample value - will be between 4.5 and 7.5: - - >>> cdf = kde(sample, h=1.5, cumulative=True) - >>> round(cdf(7.5) - cdf(4.5), 2) - 0.22 - - References - ---------- - - Kernel density estimation and its application: - https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf - - Kernel functions in common use: - https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use - - Interactive graphical demonstration and exploration: - https://demonstrations.wolfram.com/KernelDensityEstimation/ - - Kernel estimation of cumulative distribution function of a random variable with bounded support - https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf - - """ - - n = len(data) - if not n: - raise StatisticsError('Empty data sequence') - - if not isinstance(data[0], (int, float)): - raise TypeError('Data sequence must contain ints or floats') - - if h <= 0.0: - raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - - match kernel: - - case 'normal' | 'gauss': - sqrt2pi = sqrt(2 * pi) - sqrt2 = sqrt(2) - K = lambda t: exp(-1/2 * t * t) / sqrt2pi - W = lambda t: 1/2 * (1.0 + erf(t / sqrt2)) - support = None - - case 'logistic': - # 1.0 / (exp(t) + 2.0 + exp(-t)) - K = lambda t: 1/2 / (1.0 + cosh(t)) - W = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) - support = None - - case 'sigmoid': - # (2/pi) / (exp(t) + exp(-t)) - c1 = 1 / pi - c2 = 2 / pi - K = lambda t: c1 / cosh(t) - W = lambda t: c2 * atan(exp(t)) - support = None - - case 'rectangular' | 'uniform': - K = lambda t: 1/2 - W = lambda t: 1/2 * t + 1/2 - support = 1.0 - - case 'triangular': - K = lambda t: 1.0 - abs(t) - W = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 - support = 1.0 - - case 'parabolic' | 'epanechnikov': - K = lambda t: 3/4 * (1.0 - t * t) - W = lambda t: -1/4 * t**3 + 3/4 * t + 1/2 - support = 1.0 - - case 'quartic' | 'biweight': - K = lambda t: 15/16 * (1.0 - t * t) ** 2 - W = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), - (t**5, t**3, t, 1.0)) - support = 1.0 - - case 'triweight': - K = lambda t: 35/32 * (1.0 - t * t) ** 3 - W = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), - (t**7, t**5, t**3, t, 1.0)) - support = 1.0 - - case 'cosine': - c1 = pi / 4 - c2 = pi / 2 - K = lambda t: c1 * cos(c2 * t) - W = lambda t: 1/2 * sin(c2 * t) + 1/2 - support = 1.0 - - case _: - raise StatisticsError(f'Unknown kernel name: {kernel!r}') - - if support is None: - - def pdf(x): - return sum(K((x - x_i) / h) for x_i in data) / (len(data) * h) - - def cdf(x): - return sum(W((x - x_i) / h) for x_i in data) / len(data) - - else: - - sample = sorted(data) - bandwidth = h * support - - def pdf(x): - nonlocal n, sample - if len(data) != n: - sample = sorted(data) - n = len(data) - i = bisect_left(sample, x - bandwidth) - j = bisect_right(sample, x + bandwidth) - supported = sample[i : j] - return sum(K((x - x_i) / h) for x_i in supported) / (n * h) - - def cdf(x): - nonlocal n, sample - if len(data) != n: - sample = sorted(data) - n = len(data) - i = bisect_left(sample, x - bandwidth) - j = bisect_right(sample, x + bandwidth) - supported = sample[i : j] - return sum((W((x - x_i) / h) for x_i in supported), i) / n - - if cumulative: - cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' - return cdf - - else: - pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' - return pdf - - -# Notes on methods for computing quantiles -# ---------------------------------------- -# -# There is no one perfect way to compute quantiles. Here we offer -# two methods that serve common needs. Most other packages -# surveyed offered at least one or both of these two, making them -# "standard" in the sense of "widely-adopted and reproducible". -# They are also easy to explain, easy to compute manually, and have -# straight-forward interpretations that aren't surprising. - -# The default method is known as "R6", "PERCENTILE.EXC", or "expected -# value of rank order statistics". The alternative method is known as -# "R7", "PERCENTILE.INC", or "mode of rank order statistics". - -# For sample data where there is a positive probability for values -# beyond the range of the data, the R6 exclusive method is a -# reasonable choice. Consider a random sample of nine values from a -# population with a uniform distribution from 0.0 to 1.0. The -# distribution of the third ranked sample point is described by -# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and -# mean=0.300. Only the latter (which corresponds with R6) gives the -# desired cut point with 30% of the population falling below that -# value, making it comparable to a result from an inv_cdf() function. -# The R6 exclusive method is also idempotent. - -# For describing population data where the end points are known to -# be included in the data, the R7 inclusive method is a reasonable -# choice. Instead of the mean, it uses the mode of the beta -# distribution for the interior points. Per Hyndman & Fan, "One nice -# property is that the vertices of Q7(p) divide the range into n - 1 -# intervals, and exactly 100p% of the intervals lie to the left of -# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." - -# If needed, other methods could be added. However, for now, the -# position is that fewer options make for easier choices and that -# external packages can be used for anything more advanced. - -def quantiles(data, *, n=4, method='exclusive'): - """Divide *data* into *n* continuous intervals with equal probability. - - Returns a list of (n - 1) cut points separating the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate *data* in to 100 equal sized groups. - - The *data* can be any iterable containing sample. - The cut points are linearly interpolated between data points. - - If *method* is set to *inclusive*, *data* is treated as population - data. The minimum value is treated as the 0th percentile and the - maximum value is treated as the 100th percentile. - """ - if n < 1: - raise StatisticsError('n must be at least 1') - data = sorted(data) - ld = len(data) - if ld < 2: - if ld == 1: - return data * (n - 1) - raise StatisticsError('must have at least one data point') - - if method == 'inclusive': - m = ld - 1 - result = [] - for i in range(1, n): - j, delta = divmod(i * m, n) - interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n - result.append(interpolated) - return result - - if method == 'exclusive': - m = ld + 1 - result = [] - for i in range(1, n): - j = i * m // n # rescale i to m/n - j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 - delta = i*m - j*n # exact integer math - interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n - result.append(interpolated) - return result - - raise ValueError(f'Unknown method: {method!r}') - - -# === Measures of spread === - -# See http://mathworld.wolfram.com/Variance.html -# http://mathworld.wolfram.com/SampleVariance.html - - -def variance(data, xbar=None): - """Return the sample variance of data. - - data should be an iterable of Real-valued numbers, with at least two - values. The optional argument xbar, if given, should be the mean of - the data. If it is missing or None, the mean is automatically calculated. - - Use this function when your data is a sample from a population. To - calculate the variance from the entire population, see ``pvariance``. - - Examples: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as - the optional second argument ``xbar`` to avoid recalculating it: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not check that ``xbar`` is actually the mean of - ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or - impossible results. - - Decimals and Fractions are supported: + Decimals and Fractions are supported: >>> from decimal import Decimal as D >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) @@ -1144,6 +554,8 @@ def variance(data, xbar=None): Fraction(67, 108) """ + # http://mathworld.wolfram.com/SampleVariance.html + T, ss, c, n = _ss(data, xbar) if n < 2: raise StatisticsError('variance requires at least two data points') @@ -1185,6 +597,8 @@ def pvariance(data, mu=None): Fraction(13, 72) """ + # http://mathworld.wolfram.com/Variance.html + T, ss, c, n = _ss(data, mu) if n < 1: raise StatisticsError('pvariance requires at least one data point') @@ -1227,46 +641,7 @@ def pstdev(data, mu=None): return _float_sqrt_of_frac(mss.numerator, mss.denominator) -def _mean_stdev(data): - """In one pass, compute the mean and sample standard deviation as floats.""" - T, ss, xbar, n = _ss(data) - if n < 2: - raise StatisticsError('stdev requires at least two data points') - mss = ss / (n - 1) - try: - return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) - except AttributeError: - # Handle Nans and Infs gracefully - return float(xbar), float(xbar) / float(ss) - -def _sqrtprod(x: float, y: float) -> float: - "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." - h = sqrt(x * y) - if not isfinite(h): - if isinf(h) and not isinf(x) and not isinf(y): - # Finite inputs overflowed, so scale down, and recompute. - scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) - return _sqrtprod(scale * x, scale * y) / scale - return h - if not h: - if x and y: - # Non-zero inputs underflowed, so scale up, and recompute. - # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) - scale = 2.0 ** 537 - return _sqrtprod(scale * x, scale * y) / scale - return h - # Improve accuracy with a differential correction. - # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 - d = sumprod((x, h), (y, -h)) - return h + d / (2.0 * h) - - -# === Statistics for relations between two inputs === - -# See https://en.wikipedia.org/wiki/Covariance -# https://en.wikipedia.org/wiki/Pearson_correlation_coefficient -# https://en.wikipedia.org/wiki/Simple_linear_regression - +## Statistics for relations between two inputs ############################# def covariance(x, y, /): """Covariance @@ -1285,6 +660,7 @@ def covariance(x, y, /): -7.5 """ + # https://en.wikipedia.org/wiki/Covariance n = len(x) if len(y) != n: raise StatisticsError('covariance requires that both inputs have same number of data points') @@ -1318,7 +694,10 @@ def correlation(x, y, /, *, method='linear'): Spearman's rank correlation coefficient is appropriate for ordinal data or for continuous data that doesn't meet the linear proportion requirement for Pearson's correlation coefficient. + """ + # https://en.wikipedia.org/wiki/Pearson_correlation_coefficient + # https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient n = len(x) if len(y) != n: raise StatisticsError('correlation requires that both inputs have same number of data points') @@ -1326,6 +705,7 @@ def correlation(x, y, /, *, method='linear'): raise StatisticsError('correlation requires at least two data points') if method not in {'linear', 'ranked'}: raise ValueError(f'Unknown method: {method!r}') + if method == 'ranked': start = (n - 1) / -2 # Center rankings around zero x = _rank(x, start=start) @@ -1335,9 +715,11 @@ def correlation(x, y, /, *, method='linear'): ybar = fsum(y) / n x = [xi - xbar for xi in x] y = [yi - ybar for yi in y] + sxy = sumprod(x, y) sxx = sumprod(x, x) syy = sumprod(y, y) + try: return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: @@ -1384,30 +766,447 @@ def linear_regression(x, y, /, *, proportional=False): >>> linear_regression(x, y, proportional=True) #doctest: +ELLIPSIS LinearRegression(slope=2.90475..., intercept=0.0) - """ - n = len(x) - if len(y) != n: - raise StatisticsError('linear regression requires that both inputs have same number of data points') - if n < 2: - raise StatisticsError('linear regression requires at least two data points') - if not proportional: - xbar = fsum(x) / n - ybar = fsum(y) / n - x = [xi - xbar for xi in x] # List because used three times below - y = (yi - ybar for yi in y) # Generator because only used once below - sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float - sxx = sumprod(x, x) - try: - slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) - except ZeroDivisionError: - raise StatisticsError('x is constant') - intercept = 0.0 if proportional else ybar - slope * xbar - return LinearRegression(slope=slope, intercept=intercept) + """ + # https://en.wikipedia.org/wiki/Simple_linear_regression + n = len(x) + if len(y) != n: + raise StatisticsError('linear regression requires that both inputs have same number of data points') + if n < 2: + raise StatisticsError('linear regression requires at least two data points') + + if not proportional: + xbar = fsum(x) / n + ybar = fsum(y) / n + x = [xi - xbar for xi in x] # List because used three times below + y = (yi - ybar for yi in y) # Generator because only used once below + + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float + sxx = sumprod(x, x) + + try: + slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) + except ZeroDivisionError: + raise StatisticsError('x is constant') + + intercept = 0.0 if proportional else ybar - slope * xbar + return LinearRegression(slope=slope, intercept=intercept) + + +## Kernel Density Estimation ############################################### + +_kernel_specs = {} + +def register(*kernels): + "Load the kernel's pdf, cdf, invcdf, and support into _kernel_specs." + def deco(builder): + spec = dict(zip(('pdf', 'cdf', 'invcdf', 'support'), builder())) + for kernel in kernels: + _kernel_specs[kernel] = spec + return builder + return deco + +@register('normal', 'gauss') +def normal_kernel(): + sqrt2pi = sqrt(2 * pi) + sqrt2 = sqrt(2) + pdf = lambda t: exp(-1/2 * t * t) / sqrt2pi + cdf = lambda t: 1/2 * (1.0 + erf(t / sqrt2)) + invcdf = lambda t: _normal_dist_inv_cdf(t, 0.0, 1.0) + support = None + return pdf, cdf, invcdf, support + +@register('logistic') +def logistic_kernel(): + # 1.0 / (exp(t) + 2.0 + exp(-t)) + pdf = lambda t: 1/2 / (1.0 + cosh(t)) + cdf = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) + invcdf = lambda p: log(p / (1.0 - p)) + support = None + return pdf, cdf, invcdf, support + +@register('sigmoid') +def sigmoid_kernel(): + # (2/pi) / (exp(t) + exp(-t)) + c1 = 1 / pi + c2 = 2 / pi + c3 = pi / 2 + pdf = lambda t: c1 / cosh(t) + cdf = lambda t: c2 * atan(exp(t)) + invcdf = lambda p: log(tan(p * c3)) + support = None + return pdf, cdf, invcdf, support + +@register('rectangular', 'uniform') +def rectangular_kernel(): + pdf = lambda t: 1/2 + cdf = lambda t: 1/2 * t + 1/2 + invcdf = lambda p: 2.0 * p - 1.0 + support = 1.0 + return pdf, cdf, invcdf, support + +@register('triangular') +def triangular_kernel(): + pdf = lambda t: 1.0 - abs(t) + cdf = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 + invcdf = lambda p: sqrt(2.0*p) - 1.0 if p < 1/2 else 1.0 - sqrt(2.0 - 2.0*p) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('parabolic', 'epanechnikov') +def parabolic_kernel(): + pdf = lambda t: 3/4 * (1.0 - t * t) + cdf = lambda t: sumprod((-1/4, 3/4, 1/2), (t**3, t, 1.0)) + invcdf = lambda p: 2.0 * cos((acos(2.0*p - 1.0) + pi) / 3.0) + support = 1.0 + return pdf, cdf, invcdf, support + +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv + +def _quartic_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p >= 0.004 < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign + +@register('quartic', 'biweight') +def quartic_kernel(): + pdf = lambda t: 15/16 * (1.0 - t * t) ** 2 + cdf = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), + (t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_quartic_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +def _triweight_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + return x * sign + +@register('triweight') +def triweight_kernel(): + pdf = lambda t: 35/32 * (1.0 - t * t) ** 3 + cdf = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), + (t**7, t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_triweight_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('cosine') +def cosine_kernel(): + c1 = pi / 4 + c2 = pi / 2 + pdf = lambda t: c1 * cos(c2 * t) + cdf = lambda t: 1/2 * sin(c2 * t) + 1/2 + invcdf = lambda p: 2.0 * asin(2.0 * p - 1.0) / pi + support = 1.0 + return pdf, cdf, invcdf, support + +del register, normal_kernel, logistic_kernel, sigmoid_kernel +del rectangular_kernel, triangular_kernel, parabolic_kernel +del quartic_kernel, triweight_kernel, cosine_kernel + + +def kde(data, h, kernel='normal', *, cumulative=False): + """Kernel Density Estimation: Create a continuous probability density + function or cumulative distribution function from discrete samples. + + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point: + + normal (gauss) + logistic + sigmoid + + Kernels that only give weight to sample points within + the bandwidth: + + rectangular (uniform) + triangular + parabolic (epanechnikov) + quartic (biweight) + triweight + cosine + + If *cumulative* is true, will return a cumulative distribution function. + + A StatisticsError will be raised if the data sequence is empty. + + Example + ------- + + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + + Compute the area under the curve: + + >>> area = sum(f_hat(x) for x in range(-20, 20)) + >>> round(area, 4) + 1.0 + + Plot the estimated probability density function at + evenly spaced points from -6 to 10: + + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x + + Estimate P(4.5 < X <= 7.5), the probability that a new sample value + will be between 4.5 and 7.5: + + >>> cdf = kde(sample, h=1.5, cumulative=True) + >>> round(cdf(7.5) - cdf(4.5), 2) + 0.22 + + References + ---------- + + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf + + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use + + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ + + Kernel estimation of cumulative distribution function of a random variable with bounded support + https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf + + """ + + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + K = kernel_spec['pdf'] + W = kernel_spec['cdf'] + support = kernel_spec['support'] + + if support is None: + + def pdf(x): + return sum(K((x - x_i) / h) for x_i in data) / (len(data) * h) + + def cdf(x): + return sum(W((x - x_i) / h) for x_i in data) / len(data) + + else: + + sample = sorted(data) + bandwidth = h * support + + def pdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) + + def cdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum((W((x - x_i) / h) for x_i in supported), i) / n + + if cumulative: + cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' + return cdf + + else: + pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' + return pdf + + +def kde_random(data, h, kernel='normal', *, seed=None): + """Return a function that makes a random selection from the estimated + probability density function created by kde(data, h, kernel). + + Providing a *seed* allows reproducible selections within a single + thread. The seed may be an integer, float, str, or bytes. + + A StatisticsError will be raised if the *data* sequence is empty. + + Example: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + """ + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + invcdf = kernel_spec['invcdf'] + + prng = _random.Random(seed) + random = prng.random + choice = prng.choice + + def rand(): + return choice(data) + h * invcdf(random()) + + rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' + + return rand + + +## Quantiles ############################################################### + +# There is no one perfect way to compute quantiles. Here we offer +# two methods that serve common needs. Most other packages +# surveyed offered at least one or both of these two, making them +# "standard" in the sense of "widely-adopted and reproducible". +# They are also easy to explain, easy to compute manually, and have +# straight-forward interpretations that aren't surprising. + +# The default method is known as "R6", "PERCENTILE.EXC", or "expected +# value of rank order statistics". The alternative method is known as +# "R7", "PERCENTILE.INC", or "mode of rank order statistics". + +# For sample data where there is a positive probability for values +# beyond the range of the data, the R6 exclusive method is a +# reasonable choice. Consider a random sample of nine values from a +# population with a uniform distribution from 0.0 to 1.0. The +# distribution of the third ranked sample point is described by +# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and +# mean=0.300. Only the latter (which corresponds with R6) gives the +# desired cut point with 30% of the population falling below that +# value, making it comparable to a result from an inv_cdf() function. +# The R6 exclusive method is also idempotent. + +# For describing population data where the end points are known to +# be included in the data, the R7 inclusive method is a reasonable +# choice. Instead of the mean, it uses the mode of the beta +# distribution for the interior points. Per Hyndman & Fan, "One nice +# property is that the vertices of Q7(p) divide the range into n - 1 +# intervals, and exactly 100p% of the intervals lie to the left of +# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." + +# If needed, other methods could be added. However, for now, the +# position is that fewer options make for easier choices and that +# external packages can be used for anything more advanced. + +def quantiles(data, *, n=4, method='exclusive'): + """Divide *data* into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate *data* in to 100 equal sized groups. + + The *data* can be any iterable containing sample. + The cut points are linearly interpolated between data points. + + If *method* is set to *inclusive*, *data* is treated as population + data. The minimum value is treated as the 0th percentile and the + maximum value is treated as the 100th percentile. + + """ + if n < 1: + raise StatisticsError('n must be at least 1') + + data = sorted(data) + + ld = len(data) + if ld < 2: + if ld == 1: + return data * (n - 1) + raise StatisticsError('must have at least one data point') + + if method == 'inclusive': + m = ld - 1 + result = [] + for i in range(1, n): + j, delta = divmod(i * m, n) + interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n + result.append(interpolated) + return result + + if method == 'exclusive': + m = ld + 1 + result = [] + for i in range(1, n): + j = i * m // n # rescale i to m/n + j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 + delta = i*m - j*n # exact integer math + interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n + result.append(interpolated) + return result + + raise ValueError(f'Unknown method: {method!r}') ## Normal Distribution ##################################################### - def _normal_dist_inv_cdf(p, mu, sigma): # There is no closed-form solution to the inverse CDF for the normal # distribution, so we use a rational approximation instead: @@ -1415,6 +1214,7 @@ def _normal_dist_inv_cdf(p, mu, sigma): # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. q = p - 0.5 + if fabs(q) <= 0.425: r = 0.180625 - q * q # Hash sum: 55.88319_28806_14901_4439 @@ -1436,6 +1236,7 @@ def _normal_dist_inv_cdf(p, mu, sigma): 1.0) x = num / den return mu + (x * sigma) + r = p if q <= 0.0 else 1.0 - p r = sqrt(-log(r)) if r <= 5.0: @@ -1476,9 +1277,11 @@ def _normal_dist_inv_cdf(p, mu, sigma): 1.36929_88092_27358_05310e-1) * r + 5.99832_20655_58879_37690e-1) * r + 1.0) + x = num / den if q < 0.0: x = -x + return mu + (x * sigma) @@ -1635,172 +1438,416 @@ def variance(self): def __add__(x1, x2): """Add a constant or another NormalDist instance. - If *other* is a constant, translate mu by the constant, - leaving sigma unchanged. + If *other* is a constant, translate mu by the constant, + leaving sigma unchanged. + + If *other* is a NormalDist, add both the means and the variances. + Mathematically, this works only if the two distributions are + independent or if they are jointly normally distributed. + """ + if isinstance(x2, NormalDist): + return NormalDist(x1._mu + x2._mu, hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu + x2, x1._sigma) + + def __sub__(x1, x2): + """Subtract a constant or another NormalDist instance. + + If *other* is a constant, translate by the constant mu, + leaving sigma unchanged. + + If *other* is a NormalDist, subtract the means and add the variances. + Mathematically, this works only if the two distributions are + independent or if they are jointly normally distributed. + """ + if isinstance(x2, NormalDist): + return NormalDist(x1._mu - x2._mu, hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu - x2, x1._sigma) + + def __mul__(x1, x2): + """Multiply both mu and sigma by a constant. + + Used for rescaling, perhaps to change measurement units. + Sigma is scaled with the absolute value of the constant. + """ + return NormalDist(x1._mu * x2, x1._sigma * fabs(x2)) + + def __truediv__(x1, x2): + """Divide both mu and sigma by a constant. + + Used for rescaling, perhaps to change measurement units. + Sigma is scaled with the absolute value of the constant. + """ + return NormalDist(x1._mu / x2, x1._sigma / fabs(x2)) + + def __pos__(x1): + "Return a copy of the instance." + return NormalDist(x1._mu, x1._sigma) + + def __neg__(x1): + "Negates mu while keeping sigma the same." + return NormalDist(-x1._mu, x1._sigma) + + __radd__ = __add__ + + def __rsub__(x1, x2): + "Subtract a NormalDist from a constant or another NormalDist." + return -(x1 - x2) + + __rmul__ = __mul__ + + def __eq__(x1, x2): + "Two NormalDist objects are equal if their mu and sigma are both equal." + if not isinstance(x2, NormalDist): + return NotImplemented + return x1._mu == x2._mu and x1._sigma == x2._sigma + + def __hash__(self): + "NormalDist objects hash equal if their mu and sigma are both equal." + return hash((self._mu, self._sigma)) + + def __repr__(self): + return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + + def __getstate__(self): + return self._mu, self._sigma + + def __setstate__(self, state): + self._mu, self._sigma = state + + +## Private utilities ####################################################### + +def _sum(data): + """_sum(data) -> (type, sum, count) + + Return a high-precision sum of the given numeric data as a fraction, + together with the type to be converted to and the count of items. + + Examples + -------- + + >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) + (, Fraction(19, 2), 5) + + Some sources of round-off error will be avoided: + + # Built-in sum returns zero. + >>> _sum([1e50, 1, -1e50] * 1000) + (, Fraction(1000, 1), 3000) + + Fractions and Decimals are also supported: + + >>> from fractions import Fraction as F + >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) + (, Fraction(63, 20), 4) + + >>> from decimal import Decimal as D + >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] + >>> _sum(data) + (, Fraction(6963, 10000), 4) + + Mixed types are currently treated as an error, except that int is + allowed. + + """ + count = 0 + types = set() + types_add = types.add + partials = {} + partials_get = partials.get + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + partials[d] = partials_get(d, 0) + n + if None in partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + # Sum all the partial sums using builtin sum. + total = sum(Fraction(n, d) for d, n in partials.items()) + T = reduce(_coerce, types, int) # or raise TypeError + return (T, total, count) + + +def _ss(data, c=None): + """Return the exact mean and sum of square deviations of sequence data. + + Calculations are done in a single pass, allowing the input to be an iterator. + + If given *c* is used the mean; otherwise, it is calculated from the data. + Use the *c* argument with care, as it can lead to garbage results. + + """ + if c is not None: + T, ssd, count = _sum((d := x - c) * d for x in data) + return (T, ssd, c, count) + + count = 0 + types = set() + types_add = types.add + sx_partials = defaultdict(int) + sxx_partials = defaultdict(int) + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + sx_partials[d] += n + sxx_partials[d] += n * n + + if not count: + ssd = c = Fraction(0) + elif None in sx_partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + ssd = c = sx_partials[None] + assert not _isfinite(ssd) + else: + sx = sum(Fraction(n, d) for d, n in sx_partials.items()) + sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) + # This formula has poor numeric properties for floats, + # but with fractions it is exact. + ssd = (count * sxx - sx * sx) / count + c = sx / count + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, ssd, c, count) + + +def _isfinite(x): + try: + return x.is_finite() # Likely a Decimal. + except AttributeError: + return math.isfinite(x) # Coerces to float first. + + +def _coerce(T, S): + """Coerce types T and S to a common type, or raise TypeError. + + Coercion rules are currently an implementation detail. See the CoerceTest + test class in test_statistics for details. + + """ + # See http://bugs.python.org/issue24068. + assert T is not bool, "initial type T is bool" + # If the types are the same, no need to coerce anything. Put this + # first, so that the usual case (no coercion needed) happens as soon + # as possible. + if T is S: return T + # Mixed int & other coerce to the other type. + if S is int or S is bool: return T + if T is int: return S + # If one is a (strict) subclass of the other, coerce to the subclass. + if issubclass(S, T): return S + if issubclass(T, S): return T + # Ints coerce to the other type. + if issubclass(T, int): return S + if issubclass(S, int): return T + # Mixed fraction & float coerces to float (or float subclass). + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + # Any other combination is disallowed. + msg = "don't know how to coerce %s and %s" + raise TypeError(msg % (T.__name__, S.__name__)) + + +def _exact_ratio(x): + """Return Real number x to exact (numerator, denominator) pair. + + >>> _exact_ratio(0.25) + (1, 4) + + x is expected to be an int, Fraction, Decimal or float. - If *other* is a NormalDist, add both the means and the variances. - Mathematically, this works only if the two distributions are - independent or if they are jointly normally distributed. - """ - if isinstance(x2, NormalDist): - return NormalDist(x1._mu + x2._mu, hypot(x1._sigma, x2._sigma)) - return NormalDist(x1._mu + x2, x1._sigma) + """ + try: + return x.as_integer_ratio() + except AttributeError: + pass + except (OverflowError, ValueError): + # float NAN or INF. + assert not _isfinite(x) + return (x, None) - def __sub__(x1, x2): - """Subtract a constant or another NormalDist instance. + try: + # x may be an Integral ABC. + return (x.numerator, x.denominator) + except AttributeError: + msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" + raise TypeError(msg) - If *other* is a constant, translate by the constant mu, - leaving sigma unchanged. - If *other* is a NormalDist, subtract the means and add the variances. - Mathematically, this works only if the two distributions are - independent or if they are jointly normally distributed. - """ - if isinstance(x2, NormalDist): - return NormalDist(x1._mu - x2._mu, hypot(x1._sigma, x2._sigma)) - return NormalDist(x1._mu - x2, x1._sigma) +def _convert(value, T): + """Convert value to given numeric type T.""" + if type(value) is T: + # This covers the cases where T is Fraction, or where value is + # a NAN or INF (Decimal or float). + return value + if issubclass(T, int) and value.denominator != 1: + T = float + try: + # FIXME: what do we do if this overflows? + return T(value) + except TypeError: + if issubclass(T, Decimal): + return T(value.numerator) / T(value.denominator) + else: + raise - def __mul__(x1, x2): - """Multiply both mu and sigma by a constant. - Used for rescaling, perhaps to change measurement units. - Sigma is scaled with the absolute value of the constant. - """ - return NormalDist(x1._mu * x2, x1._sigma * fabs(x2)) +def _fail_neg(values, errmsg='negative value'): + """Iterate over values, failing if any are less than zero.""" + for x in values: + if x < 0: + raise StatisticsError(errmsg) + yield x - def __truediv__(x1, x2): - """Divide both mu and sigma by a constant. - Used for rescaling, perhaps to change measurement units. - Sigma is scaled with the absolute value of the constant. - """ - return NormalDist(x1._mu / x2, x1._sigma / fabs(x2)) +def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: + """Rank order a dataset. The lowest value has rank 1. - def __pos__(x1): - "Return a copy of the instance." - return NormalDist(x1._mu, x1._sigma) + Ties are averaged so that equal values receive the same rank: - def __neg__(x1): - "Negates mu while keeping sigma the same." - return NormalDist(-x1._mu, x1._sigma) + >>> data = [31, 56, 31, 25, 75, 18] + >>> _rank(data) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] - __radd__ = __add__ + The operation is idempotent: - def __rsub__(x1, x2): - "Subtract a NormalDist from a constant or another NormalDist." - return -(x1 - x2) + >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] - __rmul__ = __mul__ + It is possible to rank the data in reverse order so that the + highest value has rank 1. Also, a key-function can extract + the field to be ranked: - def __eq__(x1, x2): - "Two NormalDist objects are equal if their mu and sigma are both equal." - if not isinstance(x2, NormalDist): - return NotImplemented - return x1._mu == x2._mu and x1._sigma == x2._sigma + >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] + >>> _rank(goals, key=itemgetter(1), reverse=True) + [2.0, 1.0, 3.0] - def __hash__(self): - "NormalDist objects hash equal if their mu and sigma are both equal." - return hash((self._mu, self._sigma)) + Ranks are conventionally numbered starting from one; however, + setting *start* to zero allows the ranks to be used as array indices: - def __repr__(self): - return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] + >>> scores = [8.1, 7.3, 9.4, 8.3] + >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] + ['Bronze', 'Certificate', 'Gold', 'Silver'] - def __getstate__(self): - return self._mu, self._sigma + """ + # If this function becomes public at some point, more thought + # needs to be given to the signature. A list of ints is + # plausible when ties is "min" or "max". When ties is "average", + # either list[float] or list[Fraction] is plausible. - def __setstate__(self, state): - self._mu, self._sigma = state + # Default handling of ties matches scipy.stats.mstats.spearmanr. + if ties != 'average': + raise ValueError(f'Unknown tie resolution method: {ties!r}') + if key is not None: + data = map(key, data) + val_pos = sorted(zip(data, count()), reverse=reverse) + i = start - 1 + result = [0] * len(val_pos) + for _, g in groupby(val_pos, key=itemgetter(0)): + group = list(g) + size = len(group) + rank = i + (size + 1) / 2 + for value, orig_pos in group: + result[orig_pos] = rank + i += size + return result -## kde_random() ############################################################## +def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: + """Square root of n/m, rounded to the nearest integer using round-to-odd.""" + # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf + a = math.isqrt(n // m) + return a | (a*a*m != n) -def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): - def f_inv(y): - "Return x such that f(x) ≈ y within the specified tolerance." - x = f_inv_estimate(y) - while abs(diff := f(x) - y) > tolerance: - x -= diff / f_prime(x) - return x - return f_inv -def _quartic_invcdf_estimate(p): - sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) - x = (2.0 * p) ** 0.4258865685331 - 1.0 - if p >= 0.004 < 0.499: - x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) - return x * sign +# For 53 bit precision floats, the bit width used in +# _float_sqrt_of_frac() is 109. +_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 -_quartic_invcdf = _newton_raphson( - f_inv_estimate = _quartic_invcdf_estimate, - f = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), (t**5, t**3, t, 1.0)), - f_prime = lambda t: 15/16 * (1.0 - t * t) ** 2) -def _triweight_invcdf_estimate(p): - sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) - x = (2.0 * p) ** 0.3400218741872791 - 1.0 - return x * sign +def _float_sqrt_of_frac(n: int, m: int) -> float: + """Square root of n/m as a float, correctly rounded.""" + # See principle and proof sketch at: https://bugs.python.org/msg407078 + q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 + if q >= 0: + numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q + denominator = 1 + else: + numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) + denominator = 1 << -q + return numerator / denominator # Convert to float -_triweight_invcdf = _newton_raphson( - f_inv_estimate = _triweight_invcdf_estimate, - f = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), - (t**7, t**5, t**3, t, 1.0)), - f_prime = lambda t: 35/32 * (1.0 - t * t) ** 3) - -_kernel_invcdfs = { - 'normal': NormalDist().inv_cdf, - 'logistic': lambda p: log(p / (1 - p)), - 'sigmoid': lambda p: log(tan(p * pi/2)), - 'rectangular': lambda p: 2*p - 1, - 'parabolic': lambda p: 2 * cos((acos(2*p-1) + pi) / 3), - 'quartic': _quartic_invcdf, - 'triweight': _triweight_invcdf, - 'triangular': lambda p: sqrt(2*p) - 1 if p < 1/2 else 1 - sqrt(2 - 2*p), - 'cosine': lambda p: 2 * asin(2*p - 1) / pi, -} -_kernel_invcdfs['gauss'] = _kernel_invcdfs['normal'] -_kernel_invcdfs['uniform'] = _kernel_invcdfs['rectangular'] -_kernel_invcdfs['epanechnikov'] = _kernel_invcdfs['parabolic'] -_kernel_invcdfs['biweight'] = _kernel_invcdfs['quartic'] -def kde_random(data, h, kernel='normal', *, seed=None): - """Return a function that makes a random selection from the estimated - probability density function created by kde(data, h, kernel). +def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: + """Square root of n/m as a Decimal, correctly rounded.""" + # Premise: For decimal, computing (n/m).sqrt() can be off + # by 1 ulp from the correctly rounded result. + # Method: Check the result, moving up or down a step if needed. + if n <= 0: + if not n: + return Decimal('0.0') + n, m = -n, -m - Providing a *seed* allows reproducible selections within a single - thread. The seed may be an integer, float, str, or bytes. + root = (Decimal(n) / Decimal(m)).sqrt() + nr, dr = root.as_integer_ratio() - A StatisticsError will be raised if the *data* sequence is empty. + plus = root.next_plus() + np, dp = plus.as_integer_ratio() + # test: n / m > ((root + plus) / 2) ** 2 + if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: + return plus - Example: + minus = root.next_minus() + nm, dm = minus.as_integer_ratio() + # test: n / m < ((root + minus) / 2) ** 2 + if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: + return minus - >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> rand = kde_random(data, h=1.5, seed=8675309) - >>> new_selections = [rand() for i in range(10)] - >>> [round(x, 1) for x in new_selections] - [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + return root - """ - n = len(data) - if not n: - raise StatisticsError('Empty data sequence') - if not isinstance(data[0], (int, float)): - raise TypeError('Data sequence must contain ints or floats') +def _mean_stdev(data): + """In one pass, compute the mean and sample standard deviation as floats.""" + T, ss, xbar, n = _ss(data) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) + try: + return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) + except AttributeError: + # Handle Nans and Infs gracefully + return float(xbar), float(xbar) / float(ss) - if h <= 0.0: - raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - kernel_invcdf = _kernel_invcdfs.get(kernel) - if kernel_invcdf is None: - raise StatisticsError(f'Unknown kernel name: {kernel!r}') +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." - prng = _random.Random(seed) - random = prng.random - choice = prng.choice + h = sqrt(x * y) - def rand(): - return choice(data) + h * kernel_invcdf(random()) + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h - rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' + if not h: + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h - return rand + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index cded8aba6e8cd7..0b28459f03d86a 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2435,12 +2435,13 @@ def integrate(func, low, high, steps=10_000): self.assertGreater(f_hat(100), 0.0) def test_kde_kernel_invcdfs(self): - kernel_invcdfs = statistics._kernel_invcdfs + kernel_specs = statistics._kernel_specs kde = statistics.kde # Verify that cdf / invcdf will round trip xarr = [i/100 for i in range(-100, 101)] - for kernel, invcdf in kernel_invcdfs.items(): + for kernel, spec in kernel_specs.items(): + invcdf = spec['invcdf'] with self.subTest(kernel=kernel): cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) for x in xarr: From fd6cd621e0cce6ba2e737103d2a62b5ade90f41f Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Sun, 2 Jun 2024 14:44:29 +1000 Subject: [PATCH 324/903] gh-118934: Fix PyEval_GetLocals docs (PEP 667) (#119932) PEP 667's description of the planned changes to PyEval_GetLocals was internally inconsistent when accepted, so the docs added for gh-74929 didn't match either the current behaviour or the intended behaviour once gh-118934 is fixed. This PR updates the documentation and 3.13 What's New to match the intended behaviour (once gh-118934 is fixed). It also tidies up lingering references to `f_locals` always being a dictionary (this hasn't been true since at least when custom namespace support for class statement execution was added) --- Doc/c-api/reflection.rst | 27 ++++++++++++++++----------- Doc/reference/datamodel.rst | 4 ++-- Doc/whatsnew/3.13.rst | 36 ++++++++++++++++++++++++++++-------- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index af9a1a74ec137e..038e6977104560 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -19,23 +19,28 @@ Reflection .. deprecated:: 3.13 - To avoid creating a reference cycle in :term:`optimized scopes `, - use either :c:func:`PyEval_GetFrameLocals` to obtain the same behaviour as calling + Use either :c:func:`PyEval_GetFrameLocals` to obtain the same behaviour as calling :func:`locals` in Python code, or else call :c:func:`PyFrame_GetLocals` on the result - of :c:func:`PyEval_GetFrame` to get the same result as this function without having to - cache the proxy instance on the underlying frame. + of :c:func:`PyEval_GetFrame` to access the :attr:`~frame.f_locals` attribute of the + currently executing frame. - Return the :attr:`~frame.f_locals` attribute of the currently executing frame, + Return a mapping providing access to the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. - If the frame refers to an :term:`optimized scope`, this returns a - write-through proxy object that allows modifying the locals. - In all other cases (classes, modules, :func:`exec`, :func:`eval`) it returns - the mapping representing the frame locals directly (as described for - :func:`locals`). + Refer to :func:`locals` for details of the mapping returned at different scopes. + + As this function returns a :term:`borrowed reference`, the dictionary returned for + :term:`optimized scopes ` is cached on the frame object and will remain + alive as long as the frame object does. Unlike :c:func:`PyEval_GetFrameLocals` and + :func:`locals`, subsequent calls to this function in the same frame will update the + contents of the cached dictionary to reflect changes in the state of the local variables + rather than returning a new snapshot. .. versionchanged:: 3.13 - As part of :pep:`667`, return a proxy object for optimized scopes. + As part of :pep:`667`, :c:func:`PyFrame_GetLocals`, :func:`locals`, and + :attr:`FrameType.f_locals ` no longer make use of the shared cache + dictionary. Refer to the :ref:`What's New entry ` for + additional details. .. c:function:: PyObject* PyEval_GetGlobals(void) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 134385ed2f1860..9110060a6177e5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1347,13 +1347,13 @@ Special read-only attributes ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. * - .. attribute:: frame.f_locals - - The dictionary used by the frame to look up + - The mapping used by the frame to look up :ref:`local variables `. If the frame refers to an :term:`optimized scope`, this may return a write-through proxy object. .. versionchanged:: 3.13 - Return a proxy for functions and comprehensions. + Return a proxy for optimized scopes. * - .. attribute:: frame.f_globals - The dictionary used by the frame to look up diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index ab260bf2a2d740..66626ac06428b9 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -287,8 +287,9 @@ returns a write-through proxy to the frame's local and locally referenced nonlocal variables in these scopes, rather than returning an inconsistently updated shared ``dict`` instance with undefined runtime semantics. -See :pep:`667` for more details, including related C API changes and -deprecations. +See :pep:`667` for more details, including related C API changes and deprecations. Porting +notes are also provided below for the affected :ref:`Python APIs ` +and :ref:`C APIs `. (PEP and implementation contributed by Mark Shannon and Tian Gao in :gh:`74929`. Documentation updates provided by Guido van Rossum and @@ -2246,6 +2247,8 @@ Changes in the Python API returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. (Contributed by Serhiy Storchaka in :gh:`115961`.) +.. _pep667-porting-notes-py: + * Calling :func:`locals` in an :term:`optimized scope` now produces an independent snapshot on each call, and hence no longer implicitly updates previously returned references. Obtaining the legacy CPython behaviour now @@ -2341,15 +2344,27 @@ Changes in the C API to :c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in :gh:`115781`.) -* Calling :c:func:`PyFrame_GetLocals` or :c:func:`PyEval_GetLocals` in an - :term:`optimized scope` now returns a write-through proxy rather than a - snapshot that gets updated at ill-specified times. If a snapshot is desired, - it must be created explicitly (e.g. with :c:func:`PyDict_Copy`) or by calling - the new :c:func:`PyEval_GetFrameLocals` API. (Changed as part of :pep:`667`.) +.. _pep667-porting-notes-c: + +* The effects of mutating the dictionary returned from :c:func:`PyEval_GetLocals` in an + :term:`optimized scope` have changed. New dict entries added this way will now *only* be + visible to subsequent :c:func:`PyEval_GetLocals` calls in that frame, as + :c:func:`PyFrame_GetLocals`, :func:`locals`, and + :attr:`FrameType.f_locals ` no longer access the same underlying cached + dictionary. Changes made to entries for actual variable names and names added via the + write-through proxy interfaces will be overwritten on subsequent calls to + :c:func:`PyEval_GetLocals` in that frame. The recommended code update depends on how the + function was being used, so refer to the deprecation notice on the function for details. + (Changed as part of :pep:`667`.) + +* Calling :c:func:`PyFrame_GetLocals` in an :term:`optimized scope` now returns a + write-through proxy rather than a snapshot that gets updated at ill-specified times. + If a snapshot is desired, it must be created explicitly (e.g. with :c:func:`PyDict_Copy`) + or by calling the new :c:func:`PyEval_GetFrameLocals` API. (Changed as part of :pep:`667`.) * :c:func:`!PyFrame_FastToLocals` and :c:func:`!PyFrame_FastToLocalsWithError` no longer have any effect. Calling these functions has been redundant since - Python 3.11, when :c:func:`PyFrame_GetLocals` was first introduced. + Python 3.11, when :c:func:`PyFrame_GetLocals` was first introduced. (Changed as part of :pep:`667`.) * :c:func:`!PyFrame_LocalsToFast` no longer has any effect. Calling this function @@ -2509,6 +2524,11 @@ Deprecated C APIs :c:func:`PyWeakref_GetRef` on Python 3.12 and older. (Contributed by Victor Stinner in :gh:`105927`.) +* Deprecate the :c:func:`PyEval_GetBuiltins`, :c:func:`PyEval_GetGlobals`, and + :c:func:`PyEval_GetLocals` functions, which return a :term:`borrowed reference`. + Refer to the deprecation notices on each function for their recommended replacements. + (Soft deprecated as part of :pep:`667`.) + Pending Removal in Python 3.14 ------------------------------ From 4aed319a8eb63b205d6007c36713cacdbf1ce8a3 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 2 Jun 2024 10:27:20 +0300 Subject: [PATCH 325/903] gh-119775: Remove ability to create immutable types with mutable bases (#119776) --- Doc/whatsnew/3.14.rst | 2 ++ Lib/test/test_capi/test_misc.py | 28 ++----------------- ...-05-30-12-51-21.gh-issue-119775.CBq9IG.rst | 2 ++ Objects/typeobject.c | 16 ++++------- 4 files changed, 13 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-05-30-12-51-21.gh-issue-119775.CBq9IG.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index d443cf9bc56b98..45ffb281fcc032 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -258,3 +258,5 @@ Deprecated Removed ------- +* Creating :c:data:`immutable types ` with mutable + bases was deprecated since 3.12 and now raises a :exc:`TypeError`. diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index f3d16e4a2fc92a..0dc0b530aec971 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -777,33 +777,11 @@ def test_pytype_fromspec_with_repeated_slots(self): with self.assertRaises(SystemError): _testcapi.create_type_from_repeated_slots(variant) - @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_immutable_type_with_mutable_base(self): - # Add deprecation warning here so it's removed in 3.14 - warnings._deprecated( - 'creating immutable classes with mutable bases', remove=(3, 14)) - - class MutableBase: - def meth(self): - return 'original' - - with self.assertWarns(DeprecationWarning): - ImmutableSubclass = _testcapi.make_immutable_type_with_base( - MutableBase) - instance = ImmutableSubclass() + class MutableBase: ... - self.assertEqual(instance.meth(), 'original') - - # Cannot override the static type's method - with self.assertRaisesRegex( - TypeError, - "cannot set 'meth' attribute of immutable type"): - ImmutableSubclass.meth = lambda self: 'overridden' - self.assertEqual(instance.meth(), 'original') - - # Can change the method on the mutable base - MutableBase.meth = lambda self: 'changed' - self.assertEqual(instance.meth(), 'changed') + with self.assertRaisesRegex(TypeError, 'Creating immutable type'): + _testcapi.make_immutable_type_with_base(MutableBase) def test_pynumber_tobase(self): from _testcapi import pynumber_tobase diff --git a/Misc/NEWS.d/next/C API/2024-05-30-12-51-21.gh-issue-119775.CBq9IG.rst b/Misc/NEWS.d/next/C API/2024-05-30-12-51-21.gh-issue-119775.CBq9IG.rst new file mode 100644 index 00000000000000..c342a3814ed5db --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-30-12-51-21.gh-issue-119775.CBq9IG.rst @@ -0,0 +1,2 @@ +Creating :c:data:`immutable types ` with mutable +bases was deprecated since 3.12 and now raises a :exc:`TypeError`. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 290306cdb677e5..0095a79a2cafec 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4613,16 +4613,12 @@ _PyType_FromMetaclass_impl( goto finally; } if (!_PyType_HasFeature(b, Py_TPFLAGS_IMMUTABLETYPE)) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, - 0, - "Creating immutable type %s from mutable base %s is " - "deprecated, and slated to be disallowed in Python 3.14.", - spec->name, - b->tp_name)) - { - goto finally; - } + PyErr_Format( + PyExc_TypeError, + "Creating immutable type %s from mutable base %N", + spec->name, b + ); + goto finally; } } } From f79ffc879b919604ed5de22ece83825006cf9a17 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sun, 2 Jun 2024 10:16:49 +0100 Subject: [PATCH 326/903] gh-119740: Remove deprecated trunc delegation (#119743) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the delegation of `int` to the `__trunc__` special method: `int` will now only delegate to `__int__` and `__index__` (in that order). `__trunc__` continues to exist, but its sole purpose is to support `math.trunc`. --------- Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Serhiy Storchaka --- Doc/library/functions.rst | 11 +-- Doc/reference/datamodel.rst | 7 +- Doc/whatsnew/3.14.rst | 5 + .../pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - .../internal/pycore_runtime_init_generated.h | 1 - .../internal/pycore_unicodeobject_generated.h | 3 - Lib/test/test_int.py | 96 +------------------ Lib/test/test_long.py | 9 -- ...-05-29-18-53-43.gh-issue-119740.zP2JNM.rst | 2 + Objects/abstract.c | 32 ------- 11 files changed, 16 insertions(+), 152 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-29-18-53-43.gh-issue-119740.zP2JNM.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 7291461c69acd2..4617767a71be18 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1004,9 +1004,8 @@ are always available. They are listed here in alphabetical order. 115 If the argument defines :meth:`~object.__int__`, - ``int(x)`` returns ``x.__int__()``. If the argument defines :meth:`~object.__index__`, - it returns ``x.__index__()``. If the argument defines :meth:`~object.__trunc__`, - it returns ``x.__trunc__()``. + ``int(x)`` returns ``x.__int__()``. If the argument defines + :meth:`~object.__index__`, it returns ``x.__index__()``. For floating point numbers, this truncates towards zero. If the argument is not a number or if *base* is given, then it must be a string, @@ -1044,9 +1043,6 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.8 Falls back to :meth:`~object.__index__` if :meth:`~object.__int__` is not defined. - .. versionchanged:: 3.11 - The delegation to :meth:`~object.__trunc__` is deprecated. - .. versionchanged:: 3.11 :class:`int` string inputs and string representations can be limited to help avoid denial of service attacks. A :exc:`ValueError` is raised when @@ -1055,6 +1051,9 @@ are always available. They are listed here in alphabetical order. See the :ref:`integer string conversion length limitation ` documentation. + .. versionchanged:: 3.14 + :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. + .. function:: isinstance(object, classinfo) Return ``True`` if the *object* argument is an instance of the *classinfo* diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 9110060a6177e5..af4c585e1c3e2f 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -3127,11 +3127,8 @@ left undefined. return the value of the object truncated to an :class:`~numbers.Integral` (typically an :class:`int`). - The built-in function :func:`int` falls back to :meth:`__trunc__` if neither - :meth:`__int__` nor :meth:`__index__` is defined. - - .. versionchanged:: 3.11 - The delegation of :func:`int` to :meth:`__trunc__` is deprecated. + .. versionchanged:: 3.14 + :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. .. _context-managers: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 45ffb281fcc032..9f471d24909215 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -225,6 +225,11 @@ Others It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed by Jelle Zijlstra in :gh:`118767`.) +* The :func:`int` built-in no longer delegates to + :meth:`~object.__trunc__`. Classes that want to support conversion to + integer must implement either :meth:`~object.__int__` or + :meth:`~object.__index__`. (Contributed by Mark Dickinson in :gh:`119743`.) + Porting to Python 3.14 ====================== diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index a0f8fb71c1ff37..b9fae11dfaa85c 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -732,7 +732,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__subclasscheck__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__subclasshook__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__truediv__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__trunc__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__type_params__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_is_unpacked_typevartuple__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_prepare_subst__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 57d85020f14e05..aa66b20859a472 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -221,7 +221,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(__subclasscheck__) STRUCT_FOR_ID(__subclasshook__) STRUCT_FOR_ID(__truediv__) - STRUCT_FOR_ID(__trunc__) STRUCT_FOR_ID(__type_params__) STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__) STRUCT_FOR_ID(__typing_prepare_subst__) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index e62ebd659d30e8..b27720e9ff6ecf 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -730,7 +730,6 @@ extern "C" { INIT_ID(__subclasscheck__), \ INIT_ID(__subclasshook__), \ INIT_ID(__truediv__), \ - INIT_ID(__trunc__), \ INIT_ID(__type_params__), \ INIT_ID(__typing_is_unpacked_typevartuple__), \ INIT_ID(__typing_prepare_subst__), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 892f580e8a6846..c61c556b758769 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -504,9 +504,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__truediv__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(__trunc__); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__type_params__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index caeccbe1fed026..ce9febd741bba2 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -402,68 +402,8 @@ def __trunc__(self): class JustTrunc(base): def __trunc__(self): return 42 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(JustTrunc()), 42) - - class ExceptionalTrunc(base): - def __trunc__(self): - 1 / 0 - with self.assertRaises(ZeroDivisionError), \ - self.assertWarns(DeprecationWarning): - int(ExceptionalTrunc()) - - for trunc_result_base in (object, Classic): - class Index(trunc_result_base): - def __index__(self): - return 42 - - class TruncReturnsNonInt(base): - def __trunc__(self): - return Index() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class Intable(trunc_result_base): - def __int__(self): - return 42 - - class TruncReturnsNonIndex(base): - def __trunc__(self): - return Intable() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class NonIntegral(trunc_result_base): - def __trunc__(self): - # Check that we avoid infinite recursion. - return NonIntegral() - - class TruncReturnsNonIntegral(base): - def __trunc__(self): - return NonIntegral() - try: - with self.assertWarns(DeprecationWarning): - int(TruncReturnsNonIntegral()) - except TypeError as e: - self.assertEqual(str(e), - "__trunc__ returned non-Integral" - " (type NonIntegral)") - else: - self.fail("Failed to raise TypeError with %s" % - ((base, trunc_result_base),)) - - # Regression test for bugs.python.org/issue16060. - class BadInt(trunc_result_base): - def __int__(self): - return 42.0 - - class TruncReturnsBadInt(base): - def __trunc__(self): - return BadInt() - - with self.assertRaises(TypeError), \ - self.assertWarns(DeprecationWarning): - int(TruncReturnsBadInt()) + with self.assertRaises(TypeError): + int(JustTrunc()) def test_int_subclass_with_index(self): class MyIndex(int): @@ -514,18 +454,6 @@ class BadInt2(int): def __int__(self): return True - class TruncReturnsBadIndex: - def __trunc__(self): - return BadIndex() - - class TruncReturnsBadInt: - def __trunc__(self): - return BadInt() - - class TruncReturnsIntSubclass: - def __trunc__(self): - return True - bad_int = BadIndex() with self.assertWarns(DeprecationWarning): n = int(bad_int) @@ -549,26 +477,6 @@ def __trunc__(self): self.assertEqual(n, 1) self.assertIs(type(n), int) - bad_int = TruncReturnsBadIndex() - with self.assertWarns(DeprecationWarning): - n = int(bad_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - - bad_int = TruncReturnsBadInt() - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, int, bad_int) - - good_int = TruncReturnsIntSubclass() - with self.assertWarns(DeprecationWarning): - n = int(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - with self.assertWarns(DeprecationWarning): - n = IntSubclass(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), IntSubclass) - def test_error_message(self): def check(s, base=None): with self.assertRaises(ValueError, diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index 41b973da2c7df0..3b2e7c4e71d10d 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -386,15 +386,6 @@ def __long__(self): return 42 self.assertRaises(TypeError, int, JustLong()) - class LongTrunc: - # __long__ should be ignored in 3.x - def __long__(self): - return 42 - def __trunc__(self): - return 1729 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(LongTrunc()), 1729) - def check_float_conversion(self, n): # Check that int -> float conversion behaviour matches # that of the pure Python version above. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-29-18-53-43.gh-issue-119740.zP2JNM.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-29-18-53-43.gh-issue-119740.zP2JNM.rst new file mode 100644 index 00000000000000..111e096d262ea0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-29-18-53-43.gh-issue-119740.zP2JNM.rst @@ -0,0 +1,2 @@ +Remove the previously-deprecated delegation of :func:`int` to +:meth:`~object.__trunc__`. diff --git a/Objects/abstract.c b/Objects/abstract.c index 8357175aa5591e..200817064e3cda 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1521,7 +1521,6 @@ PyNumber_Long(PyObject *o) { PyObject *result; PyNumberMethods *m; - PyObject *trunc_func; Py_buffer view; if (o == NULL) { @@ -1563,37 +1562,6 @@ PyNumber_Long(PyObject *o) if (m && m->nb_index) { return PyNumber_Index(o); } - trunc_func = _PyObject_LookupSpecial(o, &_Py_ID(__trunc__)); - if (trunc_func) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "The delegation of int() to __trunc__ is deprecated.", 1)) { - Py_DECREF(trunc_func); - return NULL; - } - result = _PyObject_CallNoArgs(trunc_func); - Py_DECREF(trunc_func); - if (result == NULL || PyLong_CheckExact(result)) { - return result; - } - if (PyLong_Check(result)) { - Py_SETREF(result, _PyLong_Copy((PyLongObject *)result)); - return result; - } - /* __trunc__ is specified to return an Integral type, - but int() needs to return an int. */ - if (!PyIndex_Check(result)) { - PyErr_Format( - PyExc_TypeError, - "__trunc__ returned non-Integral (type %.200s)", - Py_TYPE(result)->tp_name); - Py_DECREF(result); - return NULL; - } - Py_SETREF(result, PyNumber_Index(result)); - return result; - } - if (PyErr_Occurred()) - return NULL; if (PyUnicode_Check(o)) /* The below check is done in PyLong_FromUnicodeObject(). */ From f3b89a63cbb6d46e5ed40d5cd9813cdf9189ce35 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Sun, 2 Jun 2024 10:19:02 -0400 Subject: [PATCH 327/903] gh-117657: Fix TSAN reported race in `_PyEval_IsGILEnabled`. (#119921) The GIL may be disabled concurrently with this call so we need to use a relaxed atomic load. --- Include/internal/pycore_ceval.h | 3 ++- Tools/tsan/suppressions_free_threading.txt | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index bd3ba1225f2597..26ede31b1904b4 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -145,7 +145,8 @@ extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *, static inline int _PyEval_IsGILEnabled(PyThreadState *tstate) { - return tstate->interp->ceval.gil->enabled != 0; + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + return _Py_atomic_load_int_relaxed(&gil->enabled) != 0; } // Enable or disable the GIL used by the interpreter that owns tstate, which diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index f855e9ce2698a5..78dac6ee0c9068 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -65,7 +65,6 @@ race_top:list_get_item_ref race_top:make_pending_calls race_top:set_add_entry race_top:should_intern_string -race_top:_PyEval_IsGILEnabled race_top:llist_insert_tail race_top:_Py_slot_tp_getattr_hook race_top:add_threadstate From aa9fe98e0649f0a151942914ef4e2810ca6126c2 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 2 Jun 2024 08:13:24 -0700 Subject: [PATCH 328/903] Improve documentation for typing.get_type_hints (#119928) - Explicit list of what it does that is different from "just return __annotations__" - Remove reference to PEP 563; adding the future import doesn't do anything to type aliases, and in general it will never make get_type_hints() less likely to fail. - Remove example, as the Annotated docs already have a similar example, and it's unbalanced to have one example about this one edge case but not about other behaviors of the function. Co-authored-by: Alex Waygood --- Doc/library/typing.rst | 54 ++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index a8068609fcfbe7..94de64fcf835fc 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3080,35 +3080,37 @@ Introspection helpers Return a dictionary containing type hints for a function, method, module or class object. - This is often the same as ``obj.__annotations__``. In addition, - forward references encoded as string literals are handled by evaluating - them in ``globals``, ``locals`` and (where applicable) - :ref:`type parameter ` namespaces. - For a class ``C``, return - a dictionary constructed by merging all the ``__annotations__`` along - ``C.__mro__`` in reverse order. - - The function recursively replaces all ``Annotated[T, ...]`` with ``T``, - unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for - more information). For example: - - .. testcode:: - - class Student(NamedTuple): - name: Annotated[str, 'some marker'] - - assert get_type_hints(Student) == {'name': str} - assert get_type_hints(Student, include_extras=False) == {'name': str} - assert get_type_hints(Student, include_extras=True) == { - 'name': Annotated[str, 'some marker'] - } + This is often the same as ``obj.__annotations__``, but this function makes + the following changes to the annotations dictionary: + + * Forward references encoded as string literals or :class:`ForwardRef` + objects are handled by evaluating them in *globalns*, *localns*, and + (where applicable) *obj*'s :ref:`type parameter ` namespace. + If *globalns* or *localns* is not given, appropriate namespace + dictionaries are inferred from *obj*. + * ``None`` is replaced with :class:`types.NoneType`. + * If :func:`@no_type_check ` has been applied to *obj*, an + empty dictionary is returned. + * If *obj* is a class ``C``, the function returns a dictionary that merges + annotations from ``C``'s base classes with those on ``C`` directly. This + is done by traversing ``C.__mro__`` and iteratively combining + ``__annotations__`` dictionaries. Annotations on classes appearing + earlier in the :term:`method resolution order` always take precedence over + annotations on classes appearing later in the method resolution order. + * The function recursively replaces all occurrences of ``Annotated[T, ...]`` + with ``T``, unless *include_extras* is set to ``True`` (see + :class:`Annotated` for more information). + + See also :func:`inspect.get_annotations`, a lower-level function that + returns annotations more directly. .. note:: - :func:`get_type_hints` does not work with imported - :ref:`type aliases ` that include forward references. - Enabling postponed evaluation of annotations (:pep:`563`) may remove - the need for most forward references. + If any forward references in the annotations of *obj* are not resolvable + or are not valid Python code, this function will raise an exception + such as :exc:`NameError`. For example, this can happen with imported + :ref:`type aliases ` that include forward references, + or with names imported under :data:`if TYPE_CHECKING `. .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. From bd6d4ed6454378e48dab06f50a9be0bae6baa3a2 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 2 Jun 2024 20:39:19 +0100 Subject: [PATCH 329/903] GH-119054: Add "Reading and writing files" section to pathlib docs (#119524) Add a dedicated subsection for `open()`, `read_text()`, `read_bytes()`, `write_text()` and `write_bytes()`. --- Doc/library/pathlib.rst | 163 +++++++++++++++++++++------------------- 1 file changed, 84 insertions(+), 79 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index c72d409a8eb2d6..f37bb33321fa53 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1067,6 +1067,90 @@ Querying file type and status .. versionadded:: 3.5 +Reading and writing files +^^^^^^^^^^^^^^^^^^^^^^^^^ + + +.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) + + Open the file pointed to by the path, like the built-in :func:`open` + function does:: + + >>> p = Path('setup.py') + >>> with p.open() as f: + ... f.readline() + ... + '#!/usr/bin/env python3\n' + + +.. method:: Path.read_text(encoding=None, errors=None, newline=None) + + Return the decoded contents of the pointed-to file as a string:: + + >>> p = Path('my_text_file') + >>> p.write_text('Text file contents') + 18 + >>> p.read_text() + 'Text file contents' + + The file is opened and then closed. The optional parameters have the same + meaning as in :func:`open`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.13 + The *newline* parameter was added. + + +.. method:: Path.read_bytes() + + Return the binary contents of the pointed-to file as a bytes object:: + + >>> p = Path('my_binary_file') + >>> p.write_bytes(b'Binary file contents') + 20 + >>> p.read_bytes() + b'Binary file contents' + + .. versionadded:: 3.5 + + +.. method:: Path.write_text(data, encoding=None, errors=None, newline=None) + + Open the file pointed to in text mode, write *data* to it, and close the + file:: + + >>> p = Path('my_text_file') + >>> p.write_text('Text file contents') + 18 + >>> p.read_text() + 'Text file contents' + + An existing file of the same name is overwritten. The optional parameters + have the same meaning as in :func:`open`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.10 + The *newline* parameter was added. + + +.. method:: Path.write_bytes(data) + + Open the file pointed to in bytes mode, write *data* to it, and close the + file:: + + >>> p = Path('my_binary_file') + >>> p.write_bytes(b'Binary file contents') + 20 + >>> p.read_bytes() + b'Binary file contents' + + An existing file of the same name is overwritten. + + .. versionadded:: 3.5 + + Other methods ^^^^^^^^^^^^^ @@ -1360,18 +1444,6 @@ example because the path doesn't exist). The *exist_ok* parameter was added. -.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) - - Open the file pointed to by the path, like the built-in :func:`open` - function does:: - - >>> p = Path('setup.py') - >>> with p.open() as f: - ... f.readline() - ... - '#!/usr/bin/env python3\n' - - .. method:: Path.owner(*, follow_symlinks=True) Return the name of the user owning the file. :exc:`KeyError` is raised @@ -1388,37 +1460,6 @@ example because the path doesn't exist). The *follow_symlinks* parameter was added. -.. method:: Path.read_bytes() - - Return the binary contents of the pointed-to file as a bytes object:: - - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' - - .. versionadded:: 3.5 - - -.. method:: Path.read_text(encoding=None, errors=None, newline=None) - - Return the decoded contents of the pointed-to file as a string:: - - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' - - The file is opened and then closed. The optional parameters have the same - meaning as in :func:`open`. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.13 - The *newline* parameter was added. - .. method:: Path.readlink() Return the path to which the symbolic link points (as returned by @@ -1593,42 +1634,6 @@ example because the path doesn't exist). The *missing_ok* parameter was added. -.. method:: Path.write_bytes(data) - - Open the file pointed to in bytes mode, write *data* to it, and close the - file:: - - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' - - An existing file of the same name is overwritten. - - .. versionadded:: 3.5 - - -.. method:: Path.write_text(data, encoding=None, errors=None, newline=None) - - Open the file pointed to in text mode, write *data* to it, and close the - file:: - - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' - - An existing file of the same name is overwritten. The optional parameters - have the same meaning as in :func:`open`. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.10 - The *newline* parameter was added. - - .. _pathlib-pattern-language: Pattern language From 117a8acdab997b73ada822cce97815a86f839e15 Mon Sep 17 00:00:00 2001 From: Solomon Himelbloom <7608183+TechSolomon@users.noreply.github.com> Date: Sun, 2 Jun 2024 16:43:03 -0800 Subject: [PATCH 330/903] gh-109975: What's New in Python 3.13: fix broken link for `telnetlib` alternative (#119958) --- Doc/whatsnew/3.13.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 66626ac06428b9..903de3c04b4a07 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1334,7 +1334,7 @@ PEP 594: dead batteries (and other module removals) * :mod:`!sunau`. (Contributed by Victor Stinner in :gh:`104773`.) - * :mod:`!telnetlib`, use the projects :pypi:`telnetlib3 ` or + * :mod:`!telnetlib`, use the projects :pypi:`telnetlib3` or :pypi:`Exscript` instead. (Contributed by Victor Stinner in :gh:`104773`.) From 0594a27e5f1d87d59fa8a761dd8ca9df4e42816d Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 3 Jun 2024 12:22:41 +0900 Subject: [PATCH 331/903] gh-117657: Fix data races report by TSAN unicode-hash (gh-119907) --- Objects/unicodeobject.c | 19 +++++++++++-------- Tools/tsan/suppressions_free_threading.txt | 1 - 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index eb37b478cc4de1..12782754753ef5 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1633,7 +1633,7 @@ unicode_modifiable(PyObject *unicode) assert(_PyUnicode_CHECK(unicode)); if (Py_REFCNT(unicode) != 1) return 0; - if (_PyUnicode_HASH(unicode) != -1) + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(unicode)) != -1) return 0; if (PyUnicode_CHECK_INTERNED(unicode)) return 0; @@ -10901,9 +10901,10 @@ _PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) if (PyUnicode_CHECK_INTERNED(left)) return 0; - assert(_PyUnicode_HASH(right_uni) != -1); - Py_hash_t hash = _PyUnicode_HASH(left); - if (hash != -1 && hash != _PyUnicode_HASH(right_uni)) { + Py_hash_t right_hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(right_uni)); + assert(right_hash != -1); + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(left)); + if (hash != -1 && hash != right_hash) { return 0; } @@ -11388,12 +11389,14 @@ unicode_hash(PyObject *self) #ifdef Py_DEBUG assert(_Py_HashSecret_Initialized); #endif - if (_PyUnicode_HASH(self) != -1) - return _PyUnicode_HASH(self); - + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(self)); + if (hash != -1) { + return hash; + } x = _Py_HashBytes(PyUnicode_DATA(self), PyUnicode_GET_LENGTH(self) * PyUnicode_KIND(self)); - _PyUnicode_HASH(self) = x; + + FT_ATOMIC_STORE_SSIZE_RELAXED(_PyUnicode_HASH(self), x); return x; } diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 78dac6ee0c9068..ff9c8036e92fe7 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -49,7 +49,6 @@ race_top:set_discard_entry race_top:set_inheritable race_top:start_the_world race_top:tstate_set_detached -race_top:unicode_hash race_top:Py_SET_TYPE race_top:_PyDict_CheckConsistency race_top:_PyImport_AcquireLock From 8e6321efd72d12263398994e59c5216edcada3c0 Mon Sep 17 00:00:00 2001 From: wookie184 Date: Mon, 3 Jun 2024 07:05:57 +0100 Subject: [PATCH 332/903] gh-119961: Fix test workflow status badge in README (#119962) Co-authored-by: Nikita Sobolev --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e3163c5ff636ab..7dd3660b198784 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ This is Python version 3.14.0 alpha 0 ===================================== -.. image:: https://github.com/python/cpython/workflows/Tests/badge.svg +.. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push :alt: CPython build status on GitHub Actions :target: https://github.com/python/cpython/actions From 3ea9b92086240b2f38a74c6945e7a723b480cefe Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jun 2024 08:45:20 +0200 Subject: [PATCH 333/903] gh-119396: Optimize unicode_decode_utf8_writer() (#119957) Optimize unicode_decode_utf8_writer() Take the ascii_decode() fast-path even if dest is not aligned on size_t bytes. --- Objects/unicodeobject.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 12782754753ef5..53160f1799f2cc 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4702,8 +4702,9 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) const char *p = start; #if SIZEOF_SIZE_T <= SIZEOF_VOID_P - assert(_Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)); - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { + if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T) + && _Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)) + { /* Fast path, see in STRINGLIB(utf8_decode) for an explanation. */ /* Help allocation */ @@ -4948,9 +4949,7 @@ unicode_decode_utf8_writer(_PyUnicodeWriter *writer, const char *end = s + size; Py_ssize_t decoded = 0; Py_UCS1 *dest = (Py_UCS1*)writer->data + writer->pos * writer->kind; - if (writer->kind == PyUnicode_1BYTE_KIND - && _Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)) - { + if (writer->kind == PyUnicode_1BYTE_KIND) { decoded = ascii_decode(s, end, dest); writer->pos += decoded; From 52586f930f62bd80374f0f240a4ecce0c0238174 Mon Sep 17 00:00:00 2001 From: Radislav Chugunov <52372310+chgnrdv@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:47:36 +0300 Subject: [PATCH 334/903] gh-119506: fix `_io.TextIOWrapper.write()` write during flush (#119507) Co-authored-by: Inada Naoki --- Lib/test/test_io.py | 22 +++++++++++++ ...-05-24-14-32-24.gh-issue-119506.-nMNqq.rst | 1 + Modules/_io/textio.c | 31 +++++++++++++------ 3 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-24-14-32-24.gh-issue-119506.-nMNqq.rst diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index e5cb08c2cdd04c..1ca3edac8c8dc9 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4016,6 +4016,28 @@ def write(self, data): t.write("x"*chunk_size) self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) + def test_issue119506(self): + chunk_size = 8192 + + class MockIO(self.MockRawIO): + written = False + def write(self, data): + if not self.written: + self.written = True + t.write("middle") + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf) + t.write("abc") + t.write("def") + # writing data which size >= chunk_size cause flushing buffer before write. + t.write("g" * chunk_size) + t.flush() + + self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], + buf._write_stack) + class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio diff --git a/Misc/NEWS.d/next/Library/2024-05-24-14-32-24.gh-issue-119506.-nMNqq.rst b/Misc/NEWS.d/next/Library/2024-05-24-14-32-24.gh-issue-119506.-nMNqq.rst new file mode 100644 index 00000000000000..f9b764ae0c49b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-24-14-32-24.gh-issue-119506.-nMNqq.rst @@ -0,0 +1 @@ +Fix :meth:`!io.TextIOWrapper.write` method breaks internal buffer when the method is called again during flushing internal buffer. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 9dff8eafb2560f..c162d8106ec1fd 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1719,16 +1719,26 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) bytes_len = PyBytes_GET_SIZE(b); } - if (self->pending_bytes == NULL) { - self->pending_bytes_count = 0; - self->pending_bytes = b; - } - else if (self->pending_bytes_count + bytes_len > self->chunk_size) { - // Prevent to concatenate more than chunk_size data. - if (_textiowrapper_writeflush(self) < 0) { - Py_DECREF(b); - return NULL; + // We should avoid concatinating huge data. + // Flush the buffer before adding b to the buffer if b is not small. + // https://github.com/python/cpython/issues/87426 + if (bytes_len >= self->chunk_size) { + // _textiowrapper_writeflush() calls buffer.write(). + // self->pending_bytes can be appended during buffer->write() + // or other thread. + // We need to loop until buffer becomes empty. + // https://github.com/python/cpython/issues/118138 + // https://github.com/python/cpython/issues/119506 + while (self->pending_bytes != NULL) { + if (_textiowrapper_writeflush(self) < 0) { + Py_DECREF(b); + return NULL; + } } + } + + if (self->pending_bytes == NULL) { + assert(self->pending_bytes_count == 0); self->pending_bytes = b; } else if (!PyList_CheckExact(self->pending_bytes)) { @@ -1737,6 +1747,9 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) Py_DECREF(b); return NULL; } + // Since Python 3.12, allocating GC object won't trigger GC and release + // GIL. See https://github.com/python/cpython/issues/97922 + assert(!PyList_CheckExact(self->pending_bytes)); PyList_SET_ITEM(list, 0, self->pending_bytes); PyList_SET_ITEM(list, 1, b); self->pending_bytes = list; From 84c3191954b40e090db15da49a59fcc40afe34fd Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 3 Jun 2024 10:50:29 +0300 Subject: [PATCH 335/903] gh-118827: Remove `Quoter` from `urllib.parse` (#118828) Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.14.rst | 7 +++++++ Lib/test/test_urlparse.py | 7 ------- Lib/urllib/parse.py | 8 -------- .../2024-05-09-12-33-25.gh-issue-118827.JrzHz1.rst | 3 +++ 4 files changed, 10 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-09-12-33-25.gh-issue-118827.JrzHz1.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 9f471d24909215..47f3e30942397f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -218,6 +218,13 @@ typing * Remove :class:`!typing.ByteString`. It had previously raised a :exc:`DeprecationWarning` since Python 3.12. +urllib +------ + +* Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. + It had previously raised a :exc:`DeprecationWarning` since Python 3.11. + (Contributed by Nikita Sobolev in :gh:`118827`.) + Others ------ diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 4faad733245df9..d6c83a75c1c03a 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1507,13 +1507,6 @@ def test_unwrap(self): class DeprecationTest(unittest.TestCase): - - def test_Quoter_deprecation(self): - with self.assertWarns(DeprecationWarning) as cm: - old_class = urllib.parse.Quoter - self.assertIs(old_class, urllib.parse._Quoter) - self.assertIn('Quoter will be removed', str(cm.warning)) - def test_splittype_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittype('') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 3932bb99c7e7d1..8f724f907d4217 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -822,14 +822,6 @@ def unquote_plus(string, encoding='utf-8', errors='replace'): b'_.-~') _ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE) -def __getattr__(name): - if name == 'Quoter': - warnings.warn('Deprecated in 3.11. ' - 'urllib.parse.Quoter will be removed in Python 3.14. ' - 'It was not intended to be a public API.', - DeprecationWarning, stacklevel=2) - return _Quoter - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') class _Quoter(dict): """A mapping from bytes numbers (in range(0,256)) to strings. diff --git a/Misc/NEWS.d/next/Library/2024-05-09-12-33-25.gh-issue-118827.JrzHz1.rst b/Misc/NEWS.d/next/Library/2024-05-09-12-33-25.gh-issue-118827.JrzHz1.rst new file mode 100644 index 00000000000000..40612dd93bd6da --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-12-33-25.gh-issue-118827.JrzHz1.rst @@ -0,0 +1,3 @@ +Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. It had +previously raised a :exc:`DeprecationWarning` since Python 3.11. +Patch by Nikita Sobolev. From 1e5f615086d23c71a9701abe641b5241e4345234 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 3 Jun 2024 10:52:35 +0300 Subject: [PATCH 336/903] gh-116991: Improve `pygen --help` for `python` subparser (#116992) --- Tools/peg_generator/pegen/__main__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py index 262c8a6db68f6e..0b0b4b291c2b0e 100755 --- a/Tools/peg_generator/pegen/__main__.py +++ b/Tools/peg_generator/pegen/__main__.py @@ -107,7 +107,10 @@ def generate_python_code( help="Suppress code emission for rule actions", ) -python_parser = subparsers.add_parser("python", help="Generate Python code") +python_parser = subparsers.add_parser( + "python", + help="Generate Python code, needs grammar definition with Python actions", +) python_parser.set_defaults(func=generate_python_code) python_parser.add_argument("grammar_filename", help="Grammar description") python_parser.add_argument( From 4223f1d828d3a3e1c8d803e3fdd420afd7d85faf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jun 2024 10:15:04 +0200 Subject: [PATCH 337/903] gh-119856: Support exiting help() with just "exit" (#119858) --- Lib/pydoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 5d854c50f40d6e..2ba597d01f245e 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -2007,7 +2007,7 @@ def interact(self): if (len(request) > 2 and request[0] == request[-1] in ("'", '"') and request[0] not in request[1:-1]): request = request[1:-1] - if request.lower() in ('q', 'quit'): break + if request.lower() in ('q', 'quit', 'exit'): break if request == 'help': self.intro() else: @@ -2059,7 +2059,7 @@ def intro(self): enter "modules spam". To quit this help utility and return to the interpreter, -enter "q" or "quit". +enter "q", "quit" or "exit". '''.format('%d.%d' % sys.version_info[:2])) def list(self, items, columns=4, width=80): From 70934fb46982ad2ae677cca485a730b39635919c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jun 2024 10:26:13 +0200 Subject: [PATCH 338/903] gh-112026: Deprecate _PyDict_GetItemStringWithError() function (#119855) --- Include/cpython/dictobject.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 3fd23b9313c453..e2861c963266ea 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -37,7 +37,8 @@ typedef struct { PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); -PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); +// PyDict_GetItemStringRef() can be used instead +Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); From d7fcaa73b71f4c49c1b24cac04c9b6f1cf69b944 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 3 Jun 2024 12:29:01 +0300 Subject: [PATCH 339/903] gh-119838: Treat Fraction as a real value in mixed arithmetic operations with complex (GH-119839) --- Lib/fractions.py | 4 ++-- Lib/test/test_fractions.py | 5 +---- .../Library/2024-05-31-13-56-21.gh-issue-119838.H6XHlE.rst | 3 +++ 3 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-31-13-56-21.gh-issue-119838.H6XHlE.rst diff --git a/Lib/fractions.py b/Lib/fractions.py index 95adccd86e33a0..565503911bbe97 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -668,7 +668,7 @@ def forward(a, b): elif isinstance(b, float): return fallback_operator(float(a), b) elif handle_complex and isinstance(b, complex): - return fallback_operator(complex(a), b) + return fallback_operator(float(a), b) else: return NotImplemented forward.__name__ = '__' + fallback_operator.__name__ + '__' @@ -681,7 +681,7 @@ def reverse(b, a): elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) elif handle_complex and isinstance(a, numbers.Complex): - return fallback_operator(complex(a), complex(b)) + return fallback_operator(complex(a), float(b)) else: return NotImplemented reverse.__name__ = '__r' + fallback_operator.__name__ + '__' diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 3c7780e40db096..71865f68eb0f12 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -806,10 +806,7 @@ def testMixedMultiplication(self): self.assertTypedEquals(F(3, 2) * Polar(4, 2), Polar(F(6, 1), 2)) self.assertTypedEquals(F(3, 2) * Polar(4.0, 2), Polar(6.0, 2)) self.assertTypedEquals(F(3, 2) * Rect(4, 3), Rect(F(6, 1), F(9, 2))) - with self.assertWarnsRegex(DeprecationWarning, - "argument 'real' must be a real number, not complex"): - self.assertTypedEquals(F(3, 2) * RectComplex(4, 3), - RectComplex(6.0+0j, 4.5+0j)) + self.assertTypedEquals(F(3, 2) * RectComplex(4, 3), RectComplex(6.0, 4.5)) self.assertRaises(TypeError, operator.mul, Polar(4, 2), F(3, 2)) self.assertTypedEquals(Rect(4, 3) * F(3, 2), 6.0 + 4.5j) self.assertEqual(F(3, 2) * SymbolicComplex('X'), SymbolicComplex('3/2 * X')) diff --git a/Misc/NEWS.d/next/Library/2024-05-31-13-56-21.gh-issue-119838.H6XHlE.rst b/Misc/NEWS.d/next/Library/2024-05-31-13-56-21.gh-issue-119838.H6XHlE.rst new file mode 100644 index 00000000000000..17a87327b5b1d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-31-13-56-21.gh-issue-119838.H6XHlE.rst @@ -0,0 +1,3 @@ +In mixed arithmetic operations with :class:`~fractions.Fraction` and +complex, the fraction is now converted to :class:`float` instead of +:class:`complex`. From cae4c80714e7266772025676977e2a1b98cdcd7b Mon Sep 17 00:00:00 2001 From: Awbert <119314310+SweetyAngel@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:31:02 +0300 Subject: [PATCH 340/903] gh-119968: Improved monitoring c-api docs (#119969) --- Doc/c-api/monitoring.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index ec743b98ba7024..b34035b5548f02 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -2,7 +2,7 @@ .. _monitoring: -Monitorong C API +Monitoring C API ================ Added in version 3.13. From 367adc91fb9834eb35b168048fd54705621c3f21 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:36:20 +0100 Subject: [PATCH 341/903] gh-119786: move exception handling doc to InternalDocs (#119815) --- InternalDocs/{index.md => README.md} | 4 + InternalDocs/exception_handling.md | 201 +++++++++++++++++++++++++++ Objects/exception_handling_notes.txt | 182 ------------------------ 3 files changed, 205 insertions(+), 182 deletions(-) rename InternalDocs/{index.md => README.md} (92%) create mode 100644 InternalDocs/exception_handling.md delete mode 100644 Objects/exception_handling_notes.txt diff --git a/InternalDocs/index.md b/InternalDocs/README.md similarity index 92% rename from InternalDocs/index.md rename to InternalDocs/README.md index 32b66a254bcf2c..e69e27d1542990 100644 --- a/InternalDocs/index.md +++ b/InternalDocs/README.md @@ -10,3 +10,7 @@ to hold for other implementations of the Python language. The core dev team attempts to keep this documentation up to date. If it is not, please report that through the [issue tracker](https://github.com/python/cpython/issues). + + +[Exception Handling](exception_handling.md) + diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md new file mode 100644 index 00000000000000..22d9c3bf7933f1 --- /dev/null +++ b/InternalDocs/exception_handling.md @@ -0,0 +1,201 @@ +Description of exception handling +--------------------------------- + +Python uses a technique known as "zero-cost" exception handling, which +minimizes the cost of supporting exceptions. In the common case (where +no exception is raised) the cost is reduced to zero (or close to zero). +The cost of raising an exception is increased, but not by much. + +The following code: + +``` +try: + g(0) +except: + res = "fail" + +``` + +compiles into intermediate code like the following: + +``` + RESUME 0 + + 1 SETUP_FINALLY 8 (to L1) + + 2 LOAD_NAME 0 (g) + PUSH_NULL + LOAD_CONST 0 (0) + CALL 1 + POP_TOP + POP_BLOCK + + -- L1: PUSH_EXC_INFO + + 3 POP_TOP + + 4 LOAD_CONST 1 ('fail') + STORE_NAME 1 (res) +``` + +`SETUP_FINALLY` and `POP_BLOCK` are pseudo-instructions. This means +that they can appear in intermediate code but they are not bytecode +instructions. `SETUP_FINALLY` specifies that henceforth, exceptions +are handled by the code at label L1. The `POP_BLOCK` instruction +reverses the effect of the last `SETUP` instruction, so that the +active exception handler reverts to what it was before. + +`SETUP_FINALLY` and `POP_BLOCK` have no effect when no exceptions +are raised. The idea of zero-cost exception handling is to replace +these pseudo-instructions by metadata which is stored alongside the +bytecode, and which is inspected only when an exception occurs. +This metadata is the exception table, and it is stored in the code +object's `co_exceptiontable` field. + +When the pseudo-instructions are translated into bytecode, +`SETUP_FINALLY` and `POP_BLOCK` are removed, and the exception +table is constructed, mapping each instruction to the exception +handler that covers it, if any. Instructions which are not +covered by any exception handler within the same code object's +bytecode, do not appear in the exception table at all. + +For the code object in our example above, the table has a single +entry specifying that all instructions that were between the +`SETUP_FINALLY` and the `POP_BLOCK` are covered by the exception +handler located at label `L1`. + +Handling Exceptions +------------------- + +At runtime, when an exception occurs, the interpreter looks up +the offset of the current instruction in the exception table. If +it finds a handler, control flow transfers to it. Otherwise, the +exception bubbles up to the caller, and the caller's frame is +checked for a handler covering the `CALL` instruction. This +repeats until a handler is found or the topmost frame is reached. +If no handler is found, the program terminates. During unwinding, +the traceback is constructed as each frame is added to it. + +Along with the location of an exception handler, each entry of the +exception table also contains the stack depth of the `try` instruction +and a boolean `lasti` value, which indicates whether the instruction +offset of the raising instruction should be pushed to the stack. + +Handling an exception, once an exception table entry is found, consists +of the following steps: + + 1. pop values from the stack until it matches the stack depth for the handler. + 2. if `lasti` is true, then push the offset that the exception was raised at. + 3. push the exception to the stack. + 4. jump to the target offset and resume execution. + + +Reraising Exceptions and `lasti` +-------------------------------- + +The purpose of pushing `lasti` to the stack is for cases where an exception +needs to be re-raised, and be associated with the original instruction that +raised it. This happens, for example, at the end of a `finally` block, when +any in-flight exception needs to be propagated on. As the frame's instruction +pointer now points into the finally block, a `RERAISE` instruction +(with `oparg > 0`) sets it to the `lasti` value from the stack. + +Format of the exception table +----------------------------- + +Conceptually, the exception table consists of a sequence of 5-tuples: +``` + 1. `start-offset` (inclusive) + 2. `end-offset` (exclusive) + 3. `target` + 4. `stack-depth` + 5. `push-lasti` (boolean) +``` + +All offsets and lengths are in code units, not bytes. + +We want the format to be compact, but quickly searchable. +For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. +For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. +Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. + +It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: + `start, size, target, depth, push-lasti`. + +Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each code unit takes 2 bytes. +It also happens that depth is generally quite small. + +So, we need to encode: +``` + `start` (up to 30 bits) + `size` (up to 30 bits) + `target` (up to 30 bits) + `depth` (up to ~8 bits) + `lasti` (1 bit) +``` + +We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. +Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. +Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. +The 8 bits of a byte are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. + +In addition, we combine `depth` and `lasti` into a single value, `((depth<<1)+lasti)`, before encoding. + +For example, the exception entry: +``` + `start`: 20 + `end`: 28 + `target`: 100 + `depth`: 3 + `lasti`: False +``` + +is encoded by first converting to the more compact four value form: +``` + `start`: 20 + `size`: 8 + `target`: 100 + `depth<<1+lasti`: 6 +``` + +which is then encoded as: +``` + 148 (MSB + 20 for start) + 8 (size) + 65 (Extend bit + 1) + 36 (Remainder of target, 100 == (1<<6)+36) + 6 +``` + +for a total of five bytes. + + +Script to parse the exception table +----------------------------------- + +``` +def parse_varint(iterator): + b = next(iterator) + val = b & 63 + while b&64: + val <<= 6 + b = next(iterator) + val |= b&63 + return val +``` +``` +def parse_exception_table(code): + iterator = iter(code.co_exceptiontable) + try: + while True: + start = parse_varint(iterator)*2 + length = parse_varint(iterator)*2 + end = start + length - 2 # Present as inclusive, not exclusive + target = parse_varint(iterator)*2 + dl = parse_varint(iterator) + depth = dl >> 1 + lasti = bool(dl&1) + yield start, end, target, depth, lasti + except StopIteration: + return +``` diff --git a/Objects/exception_handling_notes.txt b/Objects/exception_handling_notes.txt deleted file mode 100644 index 387ef935ce739e..00000000000000 --- a/Objects/exception_handling_notes.txt +++ /dev/null @@ -1,182 +0,0 @@ -Description of exception handling in Python 3.11 ------------------------------------------------- - -Python 3.11 uses what is known as "zero-cost" exception handling. -Prior to 3.11, exceptions were handled by a runtime stack of "blocks". - -In zero-cost exception handling, the cost of supporting exceptions is minimized. -In the common case (where no exception is raised) the cost is reduced -to zero (or close to zero). -The cost of raising an exception is increased, but not by much. - -The following code: - -def f(): - try: - g(0) - except: - return "fail" - -compiles as follows in 3.10: - - 2 0 SETUP_FINALLY 7 (to 16) - - 3 2 LOAD_GLOBAL 0 (g) - 4 LOAD_CONST 1 (0) - 6 CALL_NO_KW 1 - 8 POP_TOP - 10 POP_BLOCK - 12 LOAD_CONST 0 (None) - 14 RETURN_VALUE - - 4 >> 16 POP_TOP - 18 POP_TOP - 20 POP_TOP - - 5 22 POP_EXCEPT - 24 LOAD_CONST 3 ('fail') - 26 RETURN_VALUE - -Note the explicit instructions to push and pop from the "block" stack: -SETUP_FINALLY and POP_BLOCK. - -In 3.11, the SETUP_FINALLY and POP_BLOCK are eliminated, replaced with -a table to determine where to jump to when an exception is raised. - - 1 0 RESUME 0 - - 2 2 NOP - - 3 4 LOAD_GLOBAL 1 (g + NULL) - 16 LOAD_CONST 1 (0) - 18 PRECALL 1 - 22 CALL 1 - 32 POP_TOP - 34 LOAD_CONST 0 (None) - 36 RETURN_VALUE - >> 38 PUSH_EXC_INFO - - 4 40 POP_TOP - - 5 42 POP_EXCEPT - 44 LOAD_CONST 2 ('fail') - 46 RETURN_VALUE - >> 48 COPY 3 - 50 POP_EXCEPT - 52 RERAISE 1 -ExceptionTable: - 4 to 32 -> 38 [0] - 38 to 40 -> 48 [1] lasti - -(Note this code is from 3.11, later versions may have slightly different bytecode.) - -If an instruction raises an exception then its offset is used to find the target to jump to. -For example, the CALL at offset 22, falls into the range 4 to 32. -So, if g() raises an exception, then control jumps to offset 38. - - -Unwinding ---------- - -When an exception is raised, the current instruction offset is used to find following: -target to jump to, stack depth, and 'lasti', which determines whether the instruction -offset of the raising instruction should be pushed. - -This information is stored in the exception table, described below. - -If there is no relevant entry, the exception bubbles up to the caller. - -If there is an entry, then: - 1. pop values from the stack until it matches the stack depth for the handler. - 2. if 'lasti' is true, then push the offset that the exception was raised at. - 3. push the exception to the stack. - 4. jump to the target offset and resume execution. - - -Format of the exception table ------------------------------ - -Conceptually, the exception table consists of a sequence of 5-tuples: - 1. start-offset (inclusive) - 2. end-offset (exclusive) - 3. target - 4. stack-depth - 5. push-lasti (boolean) - -All offsets and lengths are in instructions, not bytes. - -We want the format to be compact, but quickly searchable. -For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. -For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. -Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. - -It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: - start, size, target, depth, push-lasti - -Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each instruction takes 2 bytes. -It also happens that depth is generally quite small. - -So, we need to encode: - start (up to 30 bits) - size (up to 30 bits) - target (up to 30 bits) - depth (up to ~8 bits) - lasti (1 bit) - -We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. -Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. -Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. -The 8 bits of a bit are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. - -In addition, we will combine depth and lasti into a single value, ((depth<<1)+lasti), before encoding. - -For example, the exception entry: - start: 20 - end: 28 - target: 100 - depth: 3 - lasti: False - -is encoded first by converting to the more compact four value form: - start: 20 - size: 8 - target: 100 - depth<<1+lasti: 6 - -which is then encoded as: - 148 (MSB + 20 for start) - 8 (size) - 65 (Extend bit + 1) - 36 (Remainder of target, 100 == (1<<6)+36) - 6 - -for a total of five bytes. - - - -Script to parse the exception table ------------------------------------ - -def parse_varint(iterator): - b = next(iterator) - val = b & 63 - while b&64: - val <<= 6 - b = next(iterator) - val |= b&63 - return val - -def parse_exception_table(code): - iterator = iter(code.co_exceptiontable) - try: - while True: - start = parse_varint(iterator)*2 - length = parse_varint(iterator)*2 - end = start + length - 2 # Present as inclusive, not exclusive - target = parse_varint(iterator)*2 - dl = parse_varint(iterator) - depth = dl >> 1 - lasti = bool(dl&1) - yield start, end, target, depth, lasti - except StopIteration: - return From 61d3ab32da92e70bb97a544d76ef2b837501024f Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 3 Jun 2024 15:06:31 +0300 Subject: [PATCH 342/903] gh-116560: Add PyLong_GetSign() public function (#116561) Co-authored-by: Victor Stinner --- Doc/c-api/long.rst | 13 +++++++++++++ Doc/whatsnew/3.14.rst | 3 +++ Include/cpython/longobject.h | 10 +++++++--- Lib/test/test_capi/test_long.py | 16 ++++++++++++++++ ...024-03-10-14-55-51.gh-issue-116560.x2mZaO.rst | 1 + Modules/_testcapi/long.c | 14 ++++++++++++++ Objects/longobject.c | 12 ++++++++++++ 7 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-03-10-14-55-51.gh-issue-116560.x2mZaO.rst diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 522c028cfb8d40..a0e111af5996d7 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -494,6 +494,19 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) + + Get the sign of the integer object *obj*. + + On success, set *\*sign* to the integer sign (0, -1 or +1 for zero, negative or + positive integer, respectively) and return 0. + + On failure, return -1 with an exception set. This function always succeeds + if *obj* is a :c:type:`PyLongObject` or its subtype. + + .. versionadded:: 3.14 + + .. c:function:: int PyUnstable_Long_IsCompact(const PyLongObject* op) Return 1 if *op* is compact, 0 otherwise. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 47f3e30942397f..b2dd80b64a691a 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -255,6 +255,9 @@ C API Changes New Features ------------ +* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. + (Contributed by Sergey B Kirpichev in :gh:`116560`.) + Porting to Python 3.14 ---------------------- diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 96815938c8277a..19a6722d07734a 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -55,9 +55,13 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnsignedNativeBytes(const void* buffer, PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); -// _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. -// v must not be NULL, and must be a normalized long. -// There are no error cases. +/* PyLong_GetSign. Get the sign of an integer object: + 0, -1 or +1 for zero, negative or positive integer, respectively. + + - On success, set '*sign' to the integer sign, and return 0. + - On failure, set an exception, and return -1. */ +PyAPI_FUNC(int) PyLong_GetSign(PyObject *v, int *sign); + PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); /* _PyLong_NumBits. Return the number of bits needed to represent the diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 83f894e552f983..06a29b5a0505b4 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -721,6 +721,22 @@ def test_long_fromnativebytes(self): self.assertEqual(expect_u, fromnativebytes(v_be, n, 4, 1), f"PyLong_FromNativeBytes(buffer, {n}, )") + def test_long_getsign(self): + # Test PyLong_GetSign() + getsign = _testcapi.pylong_getsign + self.assertEqual(getsign(1), 1) + self.assertEqual(getsign(123456), 1) + self.assertEqual(getsign(-2), -1) + self.assertEqual(getsign(0), 0) + self.assertEqual(getsign(True), 1) + self.assertEqual(getsign(IntSubclass(-11)), -1) + self.assertEqual(getsign(False), 0) + + self.assertRaises(TypeError, getsign, 1.0) + self.assertRaises(TypeError, getsign, Index(123)) + + # CRASHES getsign(NULL) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C API/2024-03-10-14-55-51.gh-issue-116560.x2mZaO.rst b/Misc/NEWS.d/next/C API/2024-03-10-14-55-51.gh-issue-116560.x2mZaO.rst new file mode 100644 index 00000000000000..9bcadfd9247f78 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-10-14-55-51.gh-issue-116560.x2mZaO.rst @@ -0,0 +1 @@ +Add :c:func:`PyLong_GetSign` function. Patch by Sergey B Kirpichev. diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 769c3909ea3fb1..2b5e85d5707522 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -92,6 +92,19 @@ pylong_fromnativebytes(PyObject *module, PyObject *args) return res; } + +static PyObject * +pylong_getsign(PyObject *module, PyObject *arg) +{ + int sign; + NULLABLE(arg); + if (PyLong_GetSign(arg, &sign) == -1) { + return NULL; + } + return PyLong_FromLong(sign); +} + + static PyObject * pylong_aspid(PyObject *module, PyObject *arg) { @@ -109,6 +122,7 @@ static PyMethodDef test_methods[] = { {"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS}, {"pylong_asnativebytes", pylong_asnativebytes, METH_VARARGS}, {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, + {"pylong_getsign", pylong_getsign, METH_O}, {"pylong_aspid", pylong_aspid, METH_O}, {NULL}, }; diff --git a/Objects/longobject.c b/Objects/longobject.c index 2dc2cb7a47b460..054689471e7aa9 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -770,6 +770,18 @@ _PyLong_Sign(PyObject *vv) return _PyLong_NonCompactSign(v); } +int +PyLong_GetSign(PyObject *vv, int *sign) +{ + if (!PyLong_Check(vv)) { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } + + *sign = _PyLong_Sign(vv); + return 0; +} + static int bit_length_digit(digit x) { From 4765e1fa292007f8ddc59f33454b747312506a7a Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Mon, 3 Jun 2024 15:27:44 +0200 Subject: [PATCH 343/903] gh-102511: Amend 3.13.0b1.rst (GH-119895) --- Misc/NEWS.d/3.13.0b1.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/NEWS.d/3.13.0b1.rst b/Misc/NEWS.d/3.13.0b1.rst index 525491a2603416..09b62c8377aabd 100644 --- a/Misc/NEWS.d/3.13.0b1.rst +++ b/Misc/NEWS.d/3.13.0b1.rst @@ -300,6 +300,7 @@ Improve :exc:`SyntaxError` message for empty type param brackets. .. nonce: qDEB66 .. section: Core and Builtins +Fix :func:`os.path.normpath` for UNC paths on Windows. Speed up :func:`os.path.splitroot` with a native implementation. .. From fd01271366abefa8f991e53f090387882fbd6bdd Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 3 Jun 2024 15:42:45 +0100 Subject: [PATCH 344/903] gh-119679: Ensures correct import libraries are included in Windows install packages (GH-119790) --- ...-05-30-17-39-25.gh-issue-119679.mZC87w.rst | 1 + PC/layout/main.py | 14 +++++++----- Tools/msi/freethreaded/freethreaded_files.wxs | 22 ++++++++++++------- 3 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-05-30-17-39-25.gh-issue-119679.mZC87w.rst diff --git a/Misc/NEWS.d/next/Windows/2024-05-30-17-39-25.gh-issue-119679.mZC87w.rst b/Misc/NEWS.d/next/Windows/2024-05-30-17-39-25.gh-issue-119679.mZC87w.rst new file mode 100644 index 00000000000000..db9e798d3ddcb8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-05-30-17-39-25.gh-issue-119679.mZC87w.rst @@ -0,0 +1 @@ +Ensures correct import libraries are included in Windows installs. diff --git a/PC/layout/main.py b/PC/layout/main.py index 1c4842f8588a5b..716f01097fe3b0 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -121,7 +121,7 @@ def get_tcltk_lib(ns): def get_layout(ns): - def in_build(f, dest="", new_name=None): + def in_build(f, dest="", new_name=None, no_lib=False): n, _, x = f.rpartition(".") n = new_name or n src = ns.build / f @@ -136,7 +136,7 @@ def in_build(f, dest="", new_name=None): pdb = src.with_suffix(".pdb") if pdb.is_file(): yield dest + n + ".pdb", pdb - if ns.include_dev: + if ns.include_dev and not no_lib: lib = src.with_suffix(".lib") if lib.is_file(): yield "libs/" + n + ".lib", lib @@ -202,7 +202,9 @@ def in_build(f, dest="", new_name=None): yield "LICENSE.txt", ns.build / "LICENSE.txt" - for dest, src in rglob(ns.build, "*.pyd"): + dest="" if ns.flat_dlls else "DLLs/" + + for _, src in rglob(ns.build, "*.pyd"): if ns.include_freethreaded: if not src.match("*.cp*t-win*.pyd"): continue @@ -217,14 +219,14 @@ def in_build(f, dest="", new_name=None): continue if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: continue - yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") + yield from in_build(src.name, dest=dest, no_lib=True) - for dest, src in rglob(ns.build, "*.dll"): + for _, src in rglob(ns.build, "*.dll"): if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: continue if src in EXCLUDE_FROM_DLLS: continue - yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") + yield from in_build(src.name, no_lib=True) if ns.zip_lib: zip_name = PYTHON_ZIP_NAME diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs index adaf63c69d5ade..49ecb3429ad8f3 100644 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -48,6 +48,12 @@ + + + + + + @@ -69,8 +75,14 @@ - - + + + + + + + + @@ -147,12 +159,6 @@ - - - - - - From 42a34ddb0b63e638905b01e17a7254623a0de427 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 3 Jun 2024 11:13:07 -0400 Subject: [PATCH 345/903] gh-119588: Implement zipfile.Path.is_symlink (zipp 3.19.0). (#119591) --- Doc/library/zipfile.rst | 9 +++++++ Lib/test/test_zipfile/_path/test_path.py | 27 ++++++++++++------- Lib/zipfile/_path/__init__.py | 7 +++-- ...-05-26-21-28-11.gh-issue-119588.wlLBK5.rst | 1 + 4 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-26-21-28-11.gh-issue-119588.wlLBK5.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index aad2028523dc34..a4d9a1852f8f0d 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -585,6 +585,15 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. Return ``True`` if the current context references a file. +.. method:: Path.is_symlink() + + Return ``True`` if the current context references a symbolic link. + + .. versionadded:: 3.12 + + .. versionchanged:: 3.12.4 + Prior to 3.12.4, ``is_symlink`` would unconditionally return ``False``. + .. method:: Path.exists() Return ``True`` if the current context references a file or diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index e5d2acf39a10f8..99842ffd63a64e 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -3,6 +3,7 @@ import contextlib import pathlib import pickle +import stat import sys import unittest import zipfile @@ -21,12 +22,17 @@ class itertools: Counter = Counter +def _make_link(info: zipfile.ZipInfo): # type: ignore[name-defined] + info.external_attr |= stat.S_IFLNK << 16 + + def build_alpharep_fixture(): """ Create a zip file with this structure: . ├── a.txt + ├── n.txt (-> a.txt) ├── b │ ├── c.txt │ ├── d @@ -47,6 +53,7 @@ def build_alpharep_fixture(): - multiple files in a directory (b/c, b/f) - a directory containing only a directory (g/h) - a directory with files of different extensions (j/klm) + - a symlink (n) pointing to (a) "alpha" because it uses alphabet "rep" because it's a representative example @@ -61,6 +68,9 @@ def build_alpharep_fixture(): zf.writestr("j/k.bin", b"content of k") zf.writestr("j/l.baz", b"content of l") zf.writestr("j/m.bar", b"content of m") + zf.writestr("n.txt", b"a.txt") + _make_link(zf.infolist()[-1]) + zf.filename = "alpharep.zip" return zf @@ -91,7 +101,7 @@ def zipfile_ondisk(self, alpharep): def test_iterdir_and_types(self, alpharep): root = zipfile.Path(alpharep) assert root.is_dir() - a, b, g, j = root.iterdir() + a, k, b, g, j = root.iterdir() assert a.is_file() assert b.is_dir() assert g.is_dir() @@ -111,7 +121,7 @@ def test_is_file_missing(self, alpharep): @pass_alpharep def test_iterdir_on_file(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, k, b, g, j = root.iterdir() with self.assertRaises(ValueError): a.iterdir() @@ -126,7 +136,7 @@ def test_subdir_is_dir(self, alpharep): @pass_alpharep def test_open(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, k, b, g, j = root.iterdir() with a.open(encoding="utf-8") as strm: data = strm.read() self.assertEqual(data, "content of a") @@ -230,7 +240,7 @@ def test_open_missing_directory(self, alpharep): @pass_alpharep def test_read(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, k, b, g, j = root.iterdir() assert a.read_text(encoding="utf-8") == "content of a" # Also check positional encoding arg (gh-101144). assert a.read_text("utf-8") == "content of a" @@ -296,7 +306,7 @@ def test_mutability(self, alpharep): reflect that change. """ root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, k, b, g, j = root.iterdir() alpharep.writestr('foo.txt', 'foo') alpharep.writestr('bar/baz.txt', 'baz') assert any(child.name == 'foo.txt' for child in root.iterdir()) @@ -513,12 +523,9 @@ def test_eq_hash(self, alpharep): @pass_alpharep def test_is_symlink(self, alpharep): - """ - See python/cpython#82102 for symlink support beyond this object. - """ - root = zipfile.Path(alpharep) - assert not root.is_symlink() + assert not root.joinpath('a.txt').is_symlink() + assert root.joinpath('n.txt').is_symlink() @pass_alpharep def test_relative_to(self, alpharep): diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 79ebb777354e03..f5ea18cee61930 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -5,6 +5,7 @@ import contextlib import pathlib import re +import stat import sys from .glob import Translator @@ -390,9 +391,11 @@ def match(self, path_pattern): def is_symlink(self): """ - Return whether this path is a symlink. Always false (python/cpython#82102). + Return whether this path is a symlink. """ - return False + info = self.root.getinfo(self.at) + mode = info.external_attr >> 16 + return stat.S_ISLNK(mode) def glob(self, pattern): if not pattern: diff --git a/Misc/NEWS.d/next/Library/2024-05-26-21-28-11.gh-issue-119588.wlLBK5.rst b/Misc/NEWS.d/next/Library/2024-05-26-21-28-11.gh-issue-119588.wlLBK5.rst new file mode 100644 index 00000000000000..01321d8bfe2ad5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-26-21-28-11.gh-issue-119588.wlLBK5.rst @@ -0,0 +1 @@ +``zipfile.Path.is_symlink`` now assesses if the given path is a symlink. From 6acb32fac3511c1d5500cac66f1d6397dcdab835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 3 Jun 2024 11:32:40 -0400 Subject: [PATCH 346/903] Use Cirrus M1 macOS runners for CI (GH-119979) Co-authored-by: Ee Durbin --- .github/workflows/build.yml | 8 ++++---- Lib/test/test_pyrepl/test_unix_console.py | 3 ++- Lib/test/test_pyrepl/test_windows_console.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e63737b90b72a..cde93c77a0b82e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -199,8 +199,8 @@ jobs: uses: ./.github/workflows/reusable-macos.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - # macos-14 is M1, macos-13 is Intel - os-matrix: '["macos-14", "macos-13"]' + # Cirrus is M1, macos-13 is default GHA Intel + os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-13"]' build_macos_free_threading: name: 'macOS (free-threading)' @@ -210,8 +210,8 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} free-threading: true - # macos-14-large is Intel with 12 cores (most parallelism) - os-matrix: '["macos-14"]' + # Cirrus is M1 + os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma"]' build_ubuntu: name: 'Ubuntu' diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index d0b98f17ade094..e3bbabcb0089fb 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -6,12 +6,14 @@ from unittest.mock import MagicMock, call, patch, ANY from .support import handle_all_events, code_to_events + try: from _pyrepl.console import Event from _pyrepl.unix_console import UnixConsole except ImportError: pass + def unix_console(events, **kwargs): console = UnixConsole() console.get_event = MagicMock(side_effect=events) @@ -138,7 +140,6 @@ def test_wrap(self, _os_write): _os_write.assert_any_call(ANY, b"4") con.restore() - def test_cursor_left(self, _os_write): code = "1" events = itertools.chain( diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py index e52a54d31fb5d8..4a3b2baf64a944 100644 --- a/Lib/test/test_pyrepl/test_windows_console.py +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -1,7 +1,7 @@ import sys import unittest -if sys.platform != 'win32': +if sys.platform != "win32": raise unittest.SkipTest("test only relevant on win32") From 153b118b78588209850cc2a4cbc977f193a3ab6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 3 Jun 2024 17:48:02 +0200 Subject: [PATCH 347/903] gh-119981: Use do while(0) in some symtable.c multi-line macros (#119982) --- Python/symtable.c | 97 +++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index d8240cdd11f7ea..0ee8ca36cf8df0 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -601,16 +601,17 @@ error_at_directive(PySTEntryObject *ste, PyObject *name) global: set of all symbol names explicitly declared as global */ -#define SET_SCOPE(DICT, NAME, I) { \ - PyObject *o = PyLong_FromLong(I); \ - if (!o) \ - return 0; \ - if (PyDict_SetItem((DICT), (NAME), o) < 0) { \ +#define SET_SCOPE(DICT, NAME, I) \ + do { \ + PyObject *o = PyLong_FromLong(I); \ + if (!o) \ + return 0; \ + if (PyDict_SetItem((DICT), (NAME), o) < 0) { \ + Py_DECREF(o); \ + return 0; \ + } \ Py_DECREF(o); \ - return 0; \ - } \ - Py_DECREF(o); \ -} + } while(0) /* Decide on scope of name, given flags. @@ -1562,39 +1563,45 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, return --(ST)->recursion_depth,(X) #define VISIT(ST, TYPE, V) \ - if (!symtable_visit_ ## TYPE((ST), (V))) \ - VISIT_QUIT((ST), 0); - -#define VISIT_SEQ(ST, TYPE, SEQ) { \ - int i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = 0; i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ - } \ -} - -#define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \ - int i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = (START); i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ - } \ -} - -#define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) { \ - int i = 0; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = 0; i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!elt) continue; /* can be NULL */ \ - if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ - } \ -} + do { \ + if (!symtable_visit_ ## TYPE((ST), (V))) { \ + VISIT_QUIT((ST), 0); \ + } \ + } while(0) + +#define VISIT_SEQ(ST, TYPE, SEQ) \ + do { \ + int i; \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + VISIT_QUIT((ST), 0); \ + } \ + } while(0) + +#define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) \ + do { \ + int i; \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = (START); i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + VISIT_QUIT((ST), 0); \ + } \ + } while(0) + +#define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) \ + do { \ + int i = 0; \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!elt) continue; /* can be NULL */ \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + VISIT_QUIT((ST), 0); \ + } \ + } while(0) static int symtable_record_directive(struct symtable *st, identifier name, int lineno, @@ -2261,11 +2268,11 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case Slice_kind: if (e->v.Slice.lower) - VISIT(st, expr, e->v.Slice.lower) + VISIT(st, expr, e->v.Slice.lower); if (e->v.Slice.upper) - VISIT(st, expr, e->v.Slice.upper) + VISIT(st, expr, e->v.Slice.upper); if (e->v.Slice.step) - VISIT(st, expr, e->v.Slice.step) + VISIT(st, expr, e->v.Slice.step); break; case Name_kind: if (!symtable_add_def(st, e->v.Name.id, From 1d4c2e4a877a48cdc8bcc9808d799b91c82b3757 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 3 Jun 2024 19:03:56 +0300 Subject: [PATCH 348/903] gh-119057: Use better error messages for zero division (#119066) --- Doc/howto/logging-cookbook.rst | 2 +- Lib/_pylong.py | 2 +- Lib/test/mathdata/ieee754.txt | 2 +- Lib/test/test_builtin.py | 10 ++++++++++ Lib/test/test_doctest/test_doctest.py | 8 ++++---- Lib/test/test_generators.py | 2 +- Lib/test/test_genexps.py | 2 +- .../2024-05-15-12-15-58.gh-issue-119057.P3G9G2.rst | 4 ++++ Objects/complexobject.c | 4 ++-- Objects/floatobject.c | 11 +++++------ Objects/longobject.c | 5 ++--- 11 files changed, 32 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-15-12-15-58.gh-issue-119057.P3G9G2.rst diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 60d88204b795f6..3ed2dd6251afe9 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -2950,7 +2950,7 @@ When run, this produces a file with exactly two lines: .. code-block:: none 28/01/2015 07:21:23|INFO|Sample message| - 28/01/2015 07:21:23|ERROR|ZeroDivisionError: integer division or modulo by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: integer division or modulo by zero'| + 28/01/2015 07:21:23|ERROR|ZeroDivisionError: division by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: division by zero'| While the above treatment is simplistic, it points the way to how exception information can be formatted to your liking. The :mod:`traceback` module may be diff --git a/Lib/_pylong.py b/Lib/_pylong.py index f7aabde1434725..a8bf5cd3e638a4 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -530,7 +530,7 @@ def int_divmod(a, b): Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b). """ if b == 0: - raise ZeroDivisionError + raise ZeroDivisionError('division by zero') elif b < 0: q, r = int_divmod(-a, -b) return q, -r diff --git a/Lib/test/mathdata/ieee754.txt b/Lib/test/mathdata/ieee754.txt index a8b8a0a2148f00..0bc45603b8b18a 100644 --- a/Lib/test/mathdata/ieee754.txt +++ b/Lib/test/mathdata/ieee754.txt @@ -116,7 +116,7 @@ inf >>> 0 ** -1 Traceback (most recent call last): ... -ZeroDivisionError: 0.0 cannot be raised to a negative power +ZeroDivisionError: zero to a negative power >>> pow(0, NAN) nan diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index d7ba58847a2992..9ff0f488dc4fa9 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -662,6 +662,16 @@ def test_divmod(self): self.assertAlmostEqual(result[1], exp_result[1]) self.assertRaises(TypeError, divmod) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 1, 0, + ) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 0.0, 0, + ) def test_eval(self): self.assertEqual(eval('1+1'), 2) diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 286c3ecfbc9239..b25d57ceeae6aa 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -1035,7 +1035,7 @@ def exceptions(): r""" ... >>> x = 12 ... >>> print(x//0) ... Traceback (most recent call last): - ... ZeroDivisionError: integer division or modulo by zero + ... ZeroDivisionError: division by zero ... ''' >>> test = doctest.DocTestFinder().find(f)[0] >>> doctest.DocTestRunner(verbose=False).run(test) @@ -1052,7 +1052,7 @@ def exceptions(): r""" ... >>> print('pre-exception output', x//0) ... pre-exception output ... Traceback (most recent call last): - ... ZeroDivisionError: integer division or modulo by zero + ... ZeroDivisionError: division by zero ... ''' >>> test = doctest.DocTestFinder().find(f)[0] >>> doctest.DocTestRunner(verbose=False).run(test) @@ -1063,7 +1063,7 @@ def exceptions(): r""" print('pre-exception output', x//0) Exception raised: ... - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero TestResults(failed=1, attempted=2) Exception messages may contain newlines: @@ -1258,7 +1258,7 @@ def exceptions(): r""" Exception raised: Traceback (most recent call last): ... - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero TestResults(failed=1, attempted=1) >>> _colorize.COLORIZE = save_colorize diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 6d36df2c7413e0..4598e62122b09c 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -907,7 +907,7 @@ def b(): File "", line 1, in ? File "", line 2, in g File "", line 2, in f - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero >>> next(k) # and the generator cannot be resumed Traceback (most recent call last): File "", line 1, in ? diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index 4f2d3cdcc7943e..7fb58a67368576 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -223,7 +223,7 @@ next(g) File "", line 1, in g = (10 // i for i in (5, 0, 2)) - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero >>> next(g) Traceback (most recent call last): File "", line 1, in -toplevel- diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-15-12-15-58.gh-issue-119057.P3G9G2.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-15-12-15-58.gh-issue-119057.P3G9G2.rst new file mode 100644 index 00000000000000..d252888906c348 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-15-12-15-58.gh-issue-119057.P3G9G2.rst @@ -0,0 +1,4 @@ +Improve :exc:`ZeroDivisionError` error message. +Now, all error messages are harmonized: all ``/``, ``//``, and ``%`` +operations just use "division by zero" message. +And ``0 ** -1`` operation uses "zero to a negative power". diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 59c84f1359b966..a8be266970afd0 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -523,7 +523,7 @@ complex_div(PyObject *v, PyObject *w) errno = 0; quot = _Py_c_quot(a, b); if (errno == EDOM) { - PyErr_SetString(PyExc_ZeroDivisionError, "complex division by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } return PyComplex_FromCComplex(quot); @@ -554,7 +554,7 @@ complex_pow(PyObject *v, PyObject *w, PyObject *z) _Py_ADJUST_ERANGE2(p.real, p.imag); if (errno == EDOM) { PyErr_SetString(PyExc_ZeroDivisionError, - "0.0 to a negative or complex power"); + "zero to a negative or complex power"); return NULL; } else if (errno == ERANGE) { diff --git a/Objects/floatobject.c b/Objects/floatobject.c index a5bf356cc9c7f0..2627ba80eed8ca 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -623,7 +623,7 @@ float_div(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(w, b); if (b == 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "float division by zero"); + "division by zero"); return NULL; } a = a / b; @@ -639,7 +639,7 @@ float_rem(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "float modulo by zero"); + "division by zero"); return NULL; } mod = fmod(vx, wx); @@ -704,7 +704,7 @@ float_divmod(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(v, vx); CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { - PyErr_SetString(PyExc_ZeroDivisionError, "float divmod()"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } _float_div_mod(vx, wx, &floordiv, &mod); @@ -719,7 +719,7 @@ float_floor_div(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(v, vx); CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { - PyErr_SetString(PyExc_ZeroDivisionError, "float floor division by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } _float_div_mod(vx, wx, &floordiv, &mod); @@ -788,8 +788,7 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) int iw_is_odd = DOUBLE_IS_ODD_INTEGER(iw); if (iw < 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "0.0 cannot be raised to a " - "negative power"); + "zero to a negative power"); return NULL; } /* use correct sign if iw is odd */ diff --git a/Objects/longobject.c b/Objects/longobject.c index 054689471e7aa9..ee0b2a038a2aab 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3121,8 +3121,7 @@ long_divrem(PyLongObject *a, PyLongObject *b, PyLongObject *z; if (size_b == 0) { - PyErr_SetString(PyExc_ZeroDivisionError, - "integer division or modulo by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return -1; } if (size_a < size_b || @@ -3185,7 +3184,7 @@ long_rem(PyLongObject *a, PyLongObject *b, PyLongObject **prem) if (size_b == 0) { PyErr_SetString(PyExc_ZeroDivisionError, - "integer modulo by zero"); + "division by zero"); return -1; } if (size_a < size_b || From 4e8aa32245e2d72bf558b711ccdbcee594347615 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jun 2024 18:34:36 +0200 Subject: [PATCH 349/903] gh-119727: Add --single-process option to regrtest (#119728) --- Lib/test/libregrtest/cmdline.py | 11 ++++++++ Lib/test/libregrtest/main.py | 26 ++++++++++++------- Lib/test/libregrtest/worker.py | 6 ++--- Lib/test/test_regrtest.py | 13 ++++++++++ ...-05-29-15-28-08.gh-issue-119727.dVkaZM.rst | 2 ++ 5 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index d4dac77b250ad6..2ff4715e82a41b 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -174,6 +174,7 @@ def __init__(self, **kwargs) -> None: self.tempdir = None self._add_python_opts = True self.xmlpath = None + self.single_process = False super().__init__(**kwargs) @@ -307,6 +308,12 @@ def _create_parser(): group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, help='run PROCESSES processes at once') + group.add_argument('--single-process', action='store_true', + dest='single_process', + help='always run all tests sequentially in ' + 'a single process, ignore -jN option, ' + 'and failed tests are also rerun sequentially ' + 'in the same process') group.add_argument('-T', '--coverage', action='store_true', dest='trace', help='turn on code coverage tracing using the trace ' @@ -435,6 +442,10 @@ def _parse_args(args, **kwargs): else: ns._add_python_opts = False + # --singleprocess overrides -jN option + if ns.single_process: + ns.use_mp = None + # When both --slow-ci and --fast-ci options are present, # --slow-ci has the priority if ns.slow_ci: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9e7a7d60880091..5148d3070513e8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -89,12 +89,13 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.cmdline_args: TestList = ns.args # Workers - if ns.use_mp is None: - num_workers = 0 # run sequentially + self.single_process: bool = ns.single_process + if self.single_process or ns.use_mp is None: + num_workers = 0 # run sequentially in a single process elif ns.use_mp <= 0: - num_workers = -1 # use the number of CPUs + num_workers = -1 # run in parallel, use the number of CPUs else: - num_workers = ns.use_mp + num_workers = ns.use_mp # run in parallel self.num_workers: int = num_workers self.worker_json: StrJSON | None = ns.worker_json @@ -236,7 +237,7 @@ def list_tests(tests: TestTuple): def _rerun_failed_tests(self, runtests: RunTests): # Configure the runner to re-run tests - if self.num_workers == 0: + if self.num_workers == 0 and not self.single_process: # Always run tests in fresh processes to have more deterministic # initial state. Don't re-run tests in parallel but limit to a # single worker process to have side effects (on the system load @@ -246,7 +247,6 @@ def _rerun_failed_tests(self, runtests: RunTests): tests, match_tests_dict = self.results.prepare_rerun() # Re-run failed tests - self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") runtests = runtests.copy( tests=tests, rerun=True, @@ -256,7 +256,15 @@ def _rerun_failed_tests(self, runtests: RunTests): match_tests_dict=match_tests_dict, output_on_failure=False) self.logger.set_tests(runtests) - self._run_tests_mp(runtests, self.num_workers) + + msg = f"Re-running {len(tests)} failed tests in verbose mode" + if not self.single_process: + msg = f"{msg} in subprocesses" + self.log(msg) + self._run_tests_mp(runtests, self.num_workers) + else: + self.log(msg) + self.run_tests_sequentially(runtests) return runtests def rerun_failed_tests(self, runtests: RunTests): @@ -371,7 +379,7 @@ def run_tests_sequentially(self, runtests) -> None: tests = count(jobs, 'test') else: tests = 'tests' - msg = f"Run {tests} sequentially" + msg = f"Run {tests} sequentially in a single process" if runtests.timeout: msg += " (timeout: %s)" % format_duration(runtests.timeout) self.log(msg) @@ -599,7 +607,7 @@ def _add_cross_compile_opts(self, regrtest_opts): keep_environ = True if cross_compile and hostrunner: - if self.num_workers == 0: + if self.num_workers == 0 and not self.single_process: # For now use only two cores for cross-compiled builds; # hostrunner can be expensive. regrtest_opts.extend(['-j', '2']) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 15d32b5baa04d0..86cc30835fdbda 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -14,9 +14,9 @@ USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) -NEED_TTY = set(''' - test_ioctl -'''.split()) +NEED_TTY = { + 'test_ioctl', +} def create_worker_process(runtests: WorkerRunTests, output_fd: int, diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 17eff617a56aa4..97ce797f0f6acb 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -473,6 +473,19 @@ def test_verbose3_huntrleaks(self): self.assertEqual(regrtest.hunt_refleak.runs, 10) self.assertFalse(regrtest.output_on_failure) + def test_single_process(self): + args = ['-j2', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + + args = ['--fast-ci', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + @dataclasses.dataclass(slots=True) class Rerun: diff --git a/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst b/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst new file mode 100644 index 00000000000000..bf28d8bb77b8a2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst @@ -0,0 +1,2 @@ +Add ``--single-process`` command line option to Python test runner (regrtest). +Patch by Victor Stinner. From 2e0aa731aebb8ef3d89ada82f5d39b1bbac65d1f Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Mon, 3 Jun 2024 18:07:06 +0100 Subject: [PATCH 350/903] gh-118835: pyrepl: Fix prompt length computation for custom prompts containing ANSI escape codes (#119942) --- Lib/_pyrepl/reader.py | 10 ++++-- Lib/test/test_pyrepl/test_reader.py | 32 +++++++++++++++++++ ...-06-02-15-09-17.gh-issue-118835.KUAuz6.rst | 1 + 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-02-15-09-17.gh-issue-118835.KUAuz6.rst diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 5401ae7b0ae32d..f2e68ef6f3ee66 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -28,7 +28,7 @@ from . import commands, console, input -from .utils import ANSI_ESCAPE_SEQUENCE, wlen +from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width from .trace import trace @@ -339,7 +339,8 @@ def calc_complete_screen(self) -> list[str]: screeninfo.append((0, [])) return screen - def process_prompt(self, prompt: str) -> tuple[str, int]: + @staticmethod + def process_prompt(prompt: str) -> tuple[str, int]: """Process the prompt. This means calculate the length of the prompt. The character \x01 @@ -351,6 +352,11 @@ def process_prompt(self, prompt: str) -> tuple[str, int]: # sequences if they were not explicitly within \x01...\x02. # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars, + # which breaks the logic below so we redefine it here. + def wlen(s: str) -> int: + return sum(str_width(i) for i in s) + out_prompt = "" l = wlen(prompt) pos = 0 diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index c9b03d5e711539..9fb956b655594f 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -4,6 +4,7 @@ from .support import handle_all_events, handle_events_narrow_console, code_to_events, prepare_reader from _pyrepl.console import Event +from _pyrepl.reader import Reader class TestReader(TestCase): @@ -176,3 +177,34 @@ def test_newline_within_block_trailing_whitespace(self): ) self.assert_screen_equals(reader, expected) self.assertTrue(reader.finished) + + def test_prompt_length(self): + # Handles simple ASCII prompt + ps1 = ">>> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, ps1) + self.assertEqual(l, 4) + + # Handles ANSI escape sequences + ps1 = "\033[0;32m>>> \033[0m" + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m>>> \033[0m") + self.assertEqual(l, 4) + + # Handles ANSI escape sequences bracketed in \001 .. \002 + ps1 = "\001\033[0;32m\002>>> \001\033[0m\002" + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m>>> \033[0m") + self.assertEqual(l, 4) + + # Handles wide characters in prompt + ps1 = "樂>> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, ps1) + self.assertEqual(l, 5) + + # Handles wide characters AND ANSI sequences together + ps1 = "\001\033[0;32m\002樂>\001\033[0m\002> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m樂>\033[0m> ") + self.assertEqual(l, 5) diff --git a/Misc/NEWS.d/next/Library/2024-06-02-15-09-17.gh-issue-118835.KUAuz6.rst b/Misc/NEWS.d/next/Library/2024-06-02-15-09-17.gh-issue-118835.KUAuz6.rst new file mode 100644 index 00000000000000..ec9ca20a487d76 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-02-15-09-17.gh-issue-118835.KUAuz6.rst @@ -0,0 +1 @@ +Fix _pyrepl crash when using custom prompt with ANSI escape codes. From 41c1cefbae71d687d1a935233b086473df65e15c Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 3 Jun 2024 13:42:27 -0400 Subject: [PATCH 351/903] gh-117657: Avoid `sem_clockwait` in TSAN (#119915) The `sem_clockwait` function is not currently instrumented, which leads to false positives. --- Python/parking_lot.c | 2 +- Tools/tsan/suppressions_free_threading.txt | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Python/parking_lot.c b/Python/parking_lot.c index e2def9e249cd56..841b1d71ea16cb 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -119,7 +119,7 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) if (timeout >= 0) { struct timespec ts; -#if defined(CLOCK_MONOTONIC) && defined(HAVE_SEM_CLOCKWAIT) +#if defined(CLOCK_MONOTONIC) && defined(HAVE_SEM_CLOCKWAIT) && !defined(_Py_THREAD_SANITIZER) PyTime_t now; // silently ignore error: cannot report error to the caller (void)PyTime_MonotonicRaw(&now); diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index ff9c8036e92fe7..39cc5b080c8703 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -15,14 +15,10 @@ race:set_allocator_unlocked # These entries are for warnings that trigger in a library function, as called # by a CPython function. -# https://gist.github.com/swtaarrs/9d41251e603fa6dedd604191a6da820d -race:park_detached_threads # https://gist.github.com/swtaarrs/8e0e365e1d9cecece3269a2fb2f2b8b8 race:sock_recv_impl # https://gist.github.com/swtaarrs/08dfe7883b4c975c31ecb39388987a67 race:free_threadstate -# https://gist.github.com/swtaarrs/cd6aec2006e0c1b561b68d65e9f1a872 -race:_PyParkingLot_Park # These warnings trigger directly in a CPython function. @@ -33,8 +29,6 @@ race_top:_mi_heap_delayed_free_partial race_top:_PyEval_EvalFrameDefault race_top:_PyImport_AcquireLock race_top:_PyImport_ReleaseLock -# https://gist.github.com/mpage/0a24eb2dd458441ededb498e9b0e5de8 -race_top:_PyParkingLot_Park race_top:_PyType_HasFeature race_top:assign_version_tag race_top:insertdict @@ -47,8 +41,6 @@ race_top:set_contains_key # https://gist.github.com/colesbury/d13d033f413b4ad07929d044bed86c35 race_top:set_discard_entry race_top:set_inheritable -race_top:start_the_world -race_top:tstate_set_detached race_top:Py_SET_TYPE race_top:_PyDict_CheckConsistency race_top:_PyImport_AcquireLock @@ -64,7 +56,6 @@ race_top:list_get_item_ref race_top:make_pending_calls race_top:set_add_entry race_top:should_intern_string -race_top:llist_insert_tail race_top:_Py_slot_tp_getattr_hook race_top:add_threadstate race_top:dump_traceback From b8fde5db86334690da23343f5f4326adcd8160fb Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 3 Jun 2024 21:44:36 +0100 Subject: [PATCH 352/903] update CODEOWNERS (#120003) --- .github/CODEOWNERS | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e08d6cc5719737..ca5c8aaa3a0bef 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -34,6 +34,7 @@ Python/ceval*.h @markshannon Python/compile.c @markshannon @iritkatriel Python/assemble.c @markshannon @iritkatriel Python/flowgraph.c @markshannon @iritkatriel +Python/instruction_sequence.c @iritkatriel Python/ast_opt.c @isidentical Python/bytecodes.c @markshannon Python/optimizer*.c @markshannon @@ -74,11 +75,8 @@ Programs/python.c @ericsnowcurrently Tools/build/generate_global_objects.py @ericsnowcurrently # Exceptions -Lib/traceback.py @iritkatriel Lib/test/test_except*.py @iritkatriel -Lib/test/test_traceback.py @iritkatriel Objects/exceptions.c @iritkatriel -Python/traceback.c @iritkatriel # Hashing **/*hashlib* @gpshead @tiran From 47fb4327b5c405da6df066dcaa01b7c1aefab313 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 3 Jun 2024 16:58:41 -0400 Subject: [PATCH 353/903] gh-117657: Fix race involving immortalizing objects (#119927) The free-threaded build currently immortalizes objects that use deferred reference counting (see gh-117783). This typically happens once the first non-main thread is created, but the behavior can be suppressed for tests, in subinterpreters, or during a compile() call. This fixes a race condition involving the tracking of whether the behavior is suppressed. --- Include/internal/pycore_gc.h | 14 +++++-------- Lib/test/support/__init__.py | 4 ++-- Modules/_testinternalcapi.c | 24 ++++++++-------------- Objects/codeobject.c | 4 ++-- Objects/object.c | 2 +- Python/bltinmodule.c | 6 +++--- Python/gc_free_threading.c | 14 ++++++------- Python/pystate.c | 4 +--- Tools/tsan/suppressions_free_threading.txt | 2 -- 9 files changed, 30 insertions(+), 44 deletions(-) diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 60582521db5bd7..ba8b8e1903f307 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -347,15 +347,11 @@ struct _gc_runtime_state { /* gh-117783: Deferred reference counting is not fully implemented yet, so as a temporary measure we treat objects using deferred referenence - counting as immortal. */ - struct { - /* Immortalize objects instead of marking them as using deferred - reference counting. */ - int enabled; - - /* Set enabled=1 when the first background thread is created. */ - int enable_on_thread_created; - } immortalize; + counting as immortal. The value may be zero, one, or a negative number: + 0: immortalize deferred RC objects once the first thread is created + 1: immortalize all deferred RC objects immediately + <0: suppressed; don't immortalize objects */ + int immortalize; #endif }; diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 5825efadffcb29..4b320b494bb8dd 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -529,11 +529,11 @@ def suppress_immortalization(suppress=True): yield return - old_values = _testinternalcapi.set_immortalize_deferred(False) + _testinternalcapi.suppress_immortalization(True) try: yield finally: - _testinternalcapi.set_immortalize_deferred(*old_values) + _testinternalcapi.suppress_immortalization(False) def skip_if_suppress_immortalization(): try: diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index d9b9c999603d5a..6d4a00c06ca9de 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1966,24 +1966,18 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) #endif static PyObject * -set_immortalize_deferred(PyObject *self, PyObject *value) +suppress_immortalization(PyObject *self, PyObject *value) { #ifdef Py_GIL_DISABLED - PyInterpreterState *interp = PyInterpreterState_Get(); - int old_enabled = interp->gc.immortalize.enabled; - int old_enabled_on_thread = interp->gc.immortalize.enable_on_thread_created; - int enabled_on_thread = 0; - if (!PyArg_ParseTuple(value, "i|i", - &interp->gc.immortalize.enabled, - &enabled_on_thread)) - { + int suppress = PyObject_IsTrue(value); + if (suppress < 0) { return NULL; } - interp->gc.immortalize.enable_on_thread_created = enabled_on_thread; - return Py_BuildValue("ii", old_enabled, old_enabled_on_thread); -#else - return Py_BuildValue("OO", Py_False, Py_False); + PyInterpreterState *interp = PyInterpreterState_Get(); + // Subtract two to suppress immortalization (so that 1 -> -1) + _Py_atomic_add_int(&interp->gc.immortalize, suppress ? -2 : 2); #endif + Py_RETURN_NONE; } static PyObject * @@ -1991,7 +1985,7 @@ get_immortalize_deferred(PyObject *self, PyObject *Py_UNUSED(ignored)) { #ifdef Py_GIL_DISABLED PyInterpreterState *interp = PyInterpreterState_Get(); - return PyBool_FromLong(interp->gc.immortalize.enable_on_thread_created); + return PyBool_FromLong(_Py_atomic_load_int(&interp->gc.immortalize) >= 0); #else Py_RETURN_FALSE; #endif @@ -2111,7 +2105,7 @@ static PyMethodDef module_functions[] = { #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif - {"set_immortalize_deferred", set_immortalize_deferred, METH_VARARGS}, + {"suppress_immortalization", suppress_immortalization, METH_O}, {"get_immortalize_deferred", get_immortalize_deferred, METH_NOARGS}, #ifdef _Py_TIER2 {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 2ffda8dee07c90..e3e306bfe810c4 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -110,7 +110,7 @@ should_intern_string(PyObject *o) // unless we've disabled immortalizing objects that use deferred reference // counting. PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->gc.immortalize.enable_on_thread_created) { + if (_Py_atomic_load_int(&interp->gc.immortalize) < 0) { return 1; } #endif @@ -240,7 +240,7 @@ intern_constants(PyObject *tuple, int *modified) PyThreadState *tstate = PyThreadState_GET(); if (!_Py_IsImmortal(v) && !PyCode_Check(v) && !PyUnicode_CheckExact(v) && - tstate->interp->gc.immortalize.enable_on_thread_created) + _Py_atomic_load_int(&tstate->interp->gc.immortalize) >= 0) { PyObject *interned = intern_one_constant(v); if (interned == NULL) { diff --git a/Objects/object.c b/Objects/object.c index d4fe14c5b3d1aa..5d53e9e5eaba4e 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2429,7 +2429,7 @@ _PyObject_SetDeferredRefcount(PyObject *op) assert(op->ob_ref_shared == 0); _PyObject_SET_GC_BITS(op, _PyGC_BITS_DEFERRED); PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->gc.immortalize.enabled) { + if (_Py_atomic_load_int_relaxed(&interp->gc.immortalize) == 1) { // gh-117696: immortalize objects instead of using deferred reference // counting for now. _Py_SetImmortal(op); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2a02d8161591c6..c4d3ecbeeff0e6 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -870,15 +870,15 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, // gh-118527: Disable immortalization of code constants for explicit // compile() calls to get consistent frozen outputs between the default // and free-threaded builds. + // Subtract two to suppress immortalization (so that 1 -> -1) PyInterpreterState *interp = _PyInterpreterState_GET(); - int old_value = interp->gc.immortalize.enable_on_thread_created; - interp->gc.immortalize.enable_on_thread_created = 0; + _Py_atomic_add_int(&interp->gc.immortalize, -2); #endif result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); #ifdef Py_GIL_DISABLED - interp->gc.immortalize.enable_on_thread_created = old_value; + _Py_atomic_add_int(&interp->gc.immortalize, 2); #endif Py_XDECREF(source_copy); diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index e6bd012c40ee82..d005b79ff40dbf 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -703,11 +703,9 @@ _PyGC_Init(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; - if (_Py_IsMainInterpreter(interp)) { - // gh-117783: immortalize objects that would use deferred refcounting - // once the first non-main thread is created. - gcstate->immortalize.enable_on_thread_created = 1; - } + // gh-117783: immortalize objects that would use deferred refcounting + // once the first non-main thread is created (but not in subinterpreters). + gcstate->immortalize = _Py_IsMainInterpreter(interp) ? 0 : -1; gcstate->garbage = PyList_New(0); if (gcstate->garbage == NULL) { @@ -1808,8 +1806,10 @@ _PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp) { struct visitor_args args; _PyEval_StopTheWorld(interp); - gc_visit_heaps(interp, &immortalize_visitor, &args); - interp->gc.immortalize.enabled = 1; + if (interp->gc.immortalize == 0) { + gc_visit_heaps(interp, &immortalize_visitor, &args); + interp->gc.immortalize = 1; + } _PyEval_StartTheWorld(interp); } diff --git a/Python/pystate.c b/Python/pystate.c index 36e4206b4a282e..d0293915db7689 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1583,9 +1583,7 @@ new_threadstate(PyInterpreterState *interp, int whence) } else { #ifdef Py_GIL_DISABLED - if (interp->gc.immortalize.enable_on_thread_created && - !interp->gc.immortalize.enabled) - { + if (_Py_atomic_load_int(&interp->gc.immortalize) == 0) { // Immortalize objects marked as using deferred reference counting // the first time a non-main thread is created. _PyGC_ImmortalizeDeferredObjects(interp); diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 39cc5b080c8703..d5fcac61f0db04 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -47,7 +47,6 @@ race_top:_PyImport_AcquireLock race_top:_Py_dict_lookup_threadsafe race_top:_imp_release_lock race_top:_multiprocessing_SemLock_acquire_impl -race_top:builtin_compile_impl race_top:dictiter_new race_top:dictresize race_top:insert_to_emptydict @@ -55,7 +54,6 @@ race_top:insertdict race_top:list_get_item_ref race_top:make_pending_calls race_top:set_add_entry -race_top:should_intern_string race_top:_Py_slot_tp_getattr_hook race_top:add_threadstate race_top:dump_traceback From d82a7ba041321e7b58a5a9bbc394670be6ceeb7c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 3 Jun 2024 17:56:00 -0400 Subject: [PATCH 354/903] gh-117398: Add datetime Module State (gh-119810) I was able to make use of the existing datetime_state struct, but there was one tricky thing I had to sort out. We mostly aren't converting to heap types, so we can't use things like PyType_GetModuleByDef() to look up the module state. The solution I came up with is somewhat novel, but I consider it straightforward. Also, it shouldn't have much impact on performance. In summary, this main changes here are: * I've added some macros to help hide how various objects relate to module state * as a solution to the module state lookup problem, I've stored the last loaded module on the current interpreter's internal dict (actually a weakref) * if the static type method is used after the module has been deleted, it is reloaded * to avoid extra work when loading the module, we directly copy the objects (new refs only) from the old module state into the new state if the old module hasn't been deleted yet * during module init we set various objects on the static types' __dict__s; to simplify things, we only do that the first time; once those static types have a separate __dict__ per interpreter, we'll do it every time * we now clear the module state when the module is destroyed (before, we were leaking everything in _datetime_global_state) --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Modules/_datetimemodule.c | 532 ++++++++++++------ 5 files changed, 376 insertions(+), 162 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index b9fae11dfaa85c..b186408931c92e 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -829,6 +829,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cafile)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index aa66b20859a472..e1808c85acfb2d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -318,6 +318,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(c_call) STRUCT_FOR_ID(c_exception) STRUCT_FOR_ID(c_return) + STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) STRUCT_FOR_ID(cafile) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index b27720e9ff6ecf..2dde6febc2cae4 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -827,6 +827,7 @@ extern "C" { INIT_ID(c_call), \ INIT_ID(c_exception), \ INIT_ID(c_return), \ + INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ INIT_ID(cafile), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index c61c556b758769..b00119a1bad7ff 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -795,6 +795,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(c_return); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(cached_datetime_module); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(cached_statements); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 466382b5148509..16bb4c6980aa08 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -25,17 +25,18 @@ # include /* struct timeval */ #endif + +/* forward declarations */ +static PyTypeObject PyDateTime_DateType; +static PyTypeObject PyDateTime_DateTimeType; +static PyTypeObject PyDateTime_TimeType; +static PyTypeObject PyDateTime_DeltaType; +static PyTypeObject PyDateTime_TZInfoType; +static PyTypeObject PyDateTime_TimeZoneType; + + typedef struct { - /* Static types exposed by the datetime C-API. */ - PyTypeObject *date_type; - PyTypeObject *datetime_type; - PyTypeObject *delta_type; - PyTypeObject *time_type; - PyTypeObject *tzinfo_type; - /* Exposed indirectly via TimeZone_UTC. */ - PyTypeObject *timezone_type; - - /* Other module classes. */ + /* Module heap types. */ PyTypeObject *isocalendar_date_type; /* Conversion factors. */ @@ -47,39 +48,182 @@ typedef struct { PyObject *us_per_week; // 1e6 * 3600 * 24 * 7 as Python int PyObject *seconds_per_day; // 3600 * 24 as Python int - /* The interned UTC timezone instance */ - PyObject *utc; - /* The interned Unix epoch datetime instance */ PyObject *epoch; - - /* While we use a global state, we ensure it's only initialized once */ - int initialized; } datetime_state; -static datetime_state _datetime_global_state; +/* The module has a fixed number of static objects, due to being exposed + * through the datetime C-API. There are five types exposed directly, + * one type exposed indirectly, and one singleton constant (UTC). + * + * Each of these objects is hidden behind a macro in the same way as + * the per-module objects stored in module state. The macros for the + * static objects don't need to be passed a state, but the consistency + * of doing so is more clear. We use a dedicated noop macro, NO_STATE, + * to make the special case obvious. */ + +#define NO_STATE NULL + +#define DATE_TYPE(st) &PyDateTime_DateType +#define DATETIME_TYPE(st) &PyDateTime_DateTimeType +#define TIME_TYPE(st) &PyDateTime_TimeType +#define DELTA_TYPE(st) &PyDateTime_DeltaType +#define TZINFO_TYPE(st) &PyDateTime_TZInfoType +#define TIMEZONE_TYPE(st) &PyDateTime_TimeZoneType +#define ISOCALENDAR_DATE_TYPE(st) st->isocalendar_date_type + +#define PyDate_Check(op) PyObject_TypeCheck(op, DATE_TYPE(NO_STATE)) +#define PyDate_CheckExact(op) Py_IS_TYPE(op, DATE_TYPE(NO_STATE)) + +#define PyDateTime_Check(op) PyObject_TypeCheck(op, DATETIME_TYPE(NO_STATE)) +#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, DATETIME_TYPE(NO_STATE)) + +#define PyTime_Check(op) PyObject_TypeCheck(op, TIME_TYPE(NO_STATE)) +#define PyTime_CheckExact(op) Py_IS_TYPE(op, TIME_TYPE(NO_STATE)) + +#define PyDelta_Check(op) PyObject_TypeCheck(op, DELTA_TYPE(NO_STATE)) +#define PyDelta_CheckExact(op) Py_IS_TYPE(op, DELTA_TYPE(NO_STATE)) + +#define PyTZInfo_Check(op) PyObject_TypeCheck(op, TZINFO_TYPE(NO_STATE)) +#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, TZINFO_TYPE(NO_STATE)) + +#define PyTimezone_Check(op) PyObject_TypeCheck(op, TIMEZONE_TYPE(NO_STATE)) + +#define CONST_US_PER_MS(st) st->us_per_ms +#define CONST_US_PER_SECOND(st) st->us_per_second +#define CONST_US_PER_MINUTE(st) st->us_per_minute +#define CONST_US_PER_HOUR(st) st->us_per_hour +#define CONST_US_PER_DAY(st) st->us_per_day +#define CONST_US_PER_WEEK(st) st->us_per_week +#define CONST_SEC_PER_DAY(st) st->seconds_per_day +#define CONST_EPOCH(st) st->epoch +#define CONST_UTC(st) ((PyObject *)&utc_timezone) + +static datetime_state * +get_module_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (datetime_state *)state; +} + + +#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) + +static PyObject * +get_current_module(PyInterpreterState *interp) +{ + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + return NULL; + } + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + return NULL; + } + if (ref == NULL) { + return NULL; + } + PyObject *mod = NULL; + (void)PyWeakref_GetRef(ref, &mod); + if (mod == Py_None) { + Py_CLEAR(mod); + } + Py_DECREF(ref); + return mod; +} + +static PyModuleDef datetimemodule; + +static datetime_state * +_get_current_state(PyObject **p_mod) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + PyObject *mod = get_current_module(interp); + if (mod == NULL) { + assert(!PyErr_Occurred()); + if (PyErr_Occurred()) { + return NULL; + } + /* The static types can outlive the module, + * so we must re-import the module. */ + mod = PyImport_ImportModule("_datetime"); + if (mod == NULL) { + return NULL; + } + } + datetime_state *st = get_module_state(mod); + *p_mod = mod; + return st; +} + +#define GET_CURRENT_STATE(MOD_VAR) \ + _get_current_state(&MOD_VAR) +#define RELEASE_CURRENT_STATE(ST_VAR, MOD_VAR) \ + Py_DECREF(MOD_VAR) -static inline datetime_state* get_datetime_state(void) +static int +set_current_module(PyInterpreterState *interp, PyObject *mod) { - return &_datetime_global_state; + assert(mod != NULL); + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + return -1; + } + PyObject *ref = PyWeakref_NewRef(mod, NULL); + if (ref == NULL) { + return -1; + } + int rc = PyDict_SetItem(dict, INTERP_KEY, ref); + Py_DECREF(ref); + return rc; } -#define PyDate_Check(op) PyObject_TypeCheck(op, get_datetime_state()->date_type) -#define PyDate_CheckExact(op) Py_IS_TYPE(op, get_datetime_state()->date_type) +static void +clear_current_module(PyInterpreterState *interp, PyObject *expected) +{ + PyObject *exc = PyErr_GetRaisedException(); + + PyObject *current = NULL; + + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + goto error; + } + + if (expected != NULL) { + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + goto error; + } + if (ref != NULL) { + int rc = PyWeakref_GetRef(ref, ¤t); + Py_DECREF(ref); + if (rc < 0) { + goto error; + } + if (current != expected) { + goto finally; + } + } + } -#define PyDateTime_Check(op) PyObject_TypeCheck(op, get_datetime_state()->datetime_type) -#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, get_datetime_state()->datetime_type) + if (PyDict_DelItem(dict, INTERP_KEY) < 0) { + if (!PyErr_ExceptionMatches(PyExc_KeyError)) { + goto error; + } + } -#define PyTime_Check(op) PyObject_TypeCheck(op, get_datetime_state()->time_type) -#define PyTime_CheckExact(op) Py_IS_TYPE(op, get_datetime_state()->time_type) + goto finally; -#define PyDelta_Check(op) PyObject_TypeCheck(op, get_datetime_state()->delta_type) -#define PyDelta_CheckExact(op) Py_IS_TYPE(op, get_datetime_state()->delta_type) +error: + PyErr_Print(); -#define PyTZInfo_Check(op) PyObject_TypeCheck(op, get_datetime_state()->tzinfo_type) -#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, get_datetime_state()->tzinfo_type) +finally: + Py_XDECREF(current); + PyErr_SetRaisedException(exc); +} -#define PyTimezone_Check(op) PyObject_TypeCheck(op, get_datetime_state()->timezone_type) /* We require that C int be at least 32 bits, and use int virtually * everywhere. In just a few cases we use a temp long, where a Python @@ -988,7 +1132,7 @@ new_date_ex(int year, int month, int day, PyTypeObject *type) } #define new_date(year, month, day) \ - new_date_ex(year, month, day, get_datetime_state()->date_type) + new_date_ex(year, month, day, DATE_TYPE(NO_STATE)) // Forward declaration static PyObject * @@ -998,13 +1142,12 @@ new_datetime_ex(int, int, int, int, int, int, int, PyObject *, PyTypeObject *); static PyObject * new_date_subclass_ex(int year, int month, int day, PyObject *cls) { - datetime_state *st = get_datetime_state(); PyObject *result; // We have "fast path" constructors for two subclasses: date and datetime - if ((PyTypeObject *)cls == st->date_type) { + if ((PyTypeObject *)cls == DATE_TYPE(NO_STATE)) { result = new_date_ex(year, month, day, (PyTypeObject *)cls); } - else if ((PyTypeObject *)cls == st->datetime_type) { + else if ((PyTypeObject *)cls == DATETIME_TYPE(NO_STATE)) { result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, (PyTypeObject *)cls); } @@ -1058,8 +1201,7 @@ new_datetime_ex(int year, int month, int day, int hour, int minute, } #define new_datetime(y, m, d, hh, mm, ss, us, tzinfo, fold) \ - new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, \ - get_datetime_state()->datetime_type) + new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, DATETIME_TYPE(NO_STATE)) static PyObject * call_subclass_fold(PyObject *cls, int fold, const char *format, ...) @@ -1100,9 +1242,8 @@ new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute int second, int usecond, PyObject *tzinfo, int fold, PyObject *cls) { - datetime_state *st = get_datetime_state(); PyObject* dt; - if ((PyTypeObject*)cls == st->datetime_type) { + if ((PyTypeObject*)cls == DATETIME_TYPE(NO_STATE)) { // Use the fast path constructor dt = new_datetime(year, month, day, hour, minute, second, usecond, tzinfo, fold); @@ -1163,16 +1304,15 @@ new_time_ex(int hour, int minute, int second, int usecond, return new_time_ex2(hour, minute, second, usecond, tzinfo, 0, type); } -#define new_time(hh, mm, ss, us, tzinfo, fold) \ - new_time_ex2(hh, mm, ss, us, tzinfo, fold, get_datetime_state()->time_type) +#define new_time(hh, mm, ss, us, tzinfo, fold) \ + new_time_ex2(hh, mm, ss, us, tzinfo, fold, TIME_TYPE(NO_STATE)) static PyObject * new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, PyObject *tzinfo, int fold, PyObject *cls) { PyObject *t; - datetime_state *st = get_datetime_state(); - if ((PyTypeObject*)cls == st->time_type) { + if ((PyTypeObject*)cls == TIME_TYPE(NO_STATE)) { // Use the fast path constructor t = new_time(hour, minute, second, usecond, tzinfo, fold); } @@ -1224,7 +1364,7 @@ new_delta_ex(int days, int seconds, int microseconds, int normalize, } #define new_delta(d, s, us, normalize) \ - new_delta_ex(d, s, us, normalize, get_datetime_state()->delta_type) + new_delta_ex(d, s, us, normalize, DELTA_TYPE(NO_STATE)) typedef struct @@ -1244,8 +1384,7 @@ static PyObject * create_timezone(PyObject *offset, PyObject *name) { PyDateTime_TimeZone *self; - datetime_state *st = get_datetime_state(); - PyTypeObject *type = st->timezone_type; + PyTypeObject *type = TIMEZONE_TYPE(NO_STATE); assert(offset != NULL); assert(PyDelta_Check(offset)); @@ -1267,6 +1406,7 @@ create_timezone(PyObject *offset, PyObject *name) } static int delta_bool(PyDateTime_Delta *self); +static PyDateTime_TimeZone utc_timezone; static PyObject * new_timezone(PyObject *offset, PyObject *name) @@ -1276,8 +1416,7 @@ new_timezone(PyObject *offset, PyObject *name) assert(name == NULL || PyUnicode_Check(name)); if (name == NULL && delta_bool((PyDateTime_Delta *)offset) == 0) { - datetime_state *st = get_datetime_state(); - return Py_NewRef(st->utc); + return Py_NewRef(CONST_UTC(NO_STATE)); } if ((GET_TD_DAYS(offset) == -1 && GET_TD_SECONDS(offset) == 0 && @@ -1490,8 +1629,7 @@ tzinfo_from_isoformat_results(int rv, int tzoffset, int tz_useconds) if (rv == 1) { // Create a timezone from offset in seconds (0 returns UTC) if (tzoffset == 0) { - datetime_state *st = get_datetime_state(); - return Py_NewRef(st->utc); + return Py_NewRef(CONST_UTC(NO_STATE)); } PyObject *delta = new_delta(0, tzoffset, tz_useconds, 1); @@ -1920,11 +2058,13 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *x3 = NULL; PyObject *result = NULL; + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) goto Done; - datetime_state *st = get_datetime_state(); - x2 = PyNumber_Multiply(x1, st->seconds_per_day); /* days in seconds */ + x2 = PyNumber_Multiply(x1, CONST_SEC_PER_DAY(st)); /* days in seconds */ if (x2 == NULL) goto Done; Py_SETREF(x1, NULL); @@ -1941,7 +2081,7 @@ delta_to_microseconds(PyDateTime_Delta *self) /* x1 = */ x2 = NULL; /* x3 has days+seconds in seconds */ - x1 = PyNumber_Multiply(x3, st->us_per_second); /* us */ + x1 = PyNumber_Multiply(x3, CONST_US_PER_SECOND(st)); /* us */ if (x1 == NULL) goto Done; Py_SETREF(x3, NULL); @@ -1957,6 +2097,7 @@ delta_to_microseconds(PyDateTime_Delta *self) Py_XDECREF(x1); Py_XDECREF(x2); Py_XDECREF(x3); + RELEASE_CURRENT_STATE(st, current_mod); return result; } @@ -1996,8 +2137,10 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - datetime_state *st = get_datetime_state(); - tuple = checked_divmod(pyus, st->us_per_second); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { goto Done; } @@ -2015,7 +2158,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) num = Py_NewRef(PyTuple_GET_ITEM(tuple, 0)); /* leftover seconds */ Py_DECREF(tuple); - tuple = checked_divmod(num, st->seconds_per_day); + tuple = checked_divmod(num, CONST_SEC_PER_DAY(st)); if (tuple == NULL) goto Done; Py_DECREF(num); @@ -2040,6 +2183,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) Done: Py_XDECREF(tuple); Py_XDECREF(num); + RELEASE_CURRENT_STATE(st, current_mod); return result; BadDivmod: @@ -2049,7 +2193,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) } #define microseconds_to_delta(pymicros) \ - microseconds_to_delta_ex(pymicros, get_datetime_state()->delta_type) + microseconds_to_delta_ex(pymicros, DELTA_TYPE(NO_STATE)) static PyObject * multiply_int_timedelta(PyObject *intobj, PyDateTime_Delta *delta) @@ -2577,6 +2721,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *self = NULL; + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + /* Argument objects. */ PyObject *day = NULL; PyObject *second = NULL; @@ -2615,29 +2762,28 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) y = accum("microseconds", x, us, _PyLong_GetOne(), &leftover_us); CLEANUP; } - datetime_state *st = get_datetime_state(); if (ms) { - y = accum("milliseconds", x, ms, st->us_per_ms, &leftover_us); + y = accum("milliseconds", x, ms, CONST_US_PER_MS(st), &leftover_us); CLEANUP; } if (second) { - y = accum("seconds", x, second, st->us_per_second, &leftover_us); + y = accum("seconds", x, second, CONST_US_PER_SECOND(st), &leftover_us); CLEANUP; } if (minute) { - y = accum("minutes", x, minute, st->us_per_minute, &leftover_us); + y = accum("minutes", x, minute, CONST_US_PER_MINUTE(st), &leftover_us); CLEANUP; } if (hour) { - y = accum("hours", x, hour, st->us_per_hour, &leftover_us); + y = accum("hours", x, hour, CONST_US_PER_HOUR(st), &leftover_us); CLEANUP; } if (day) { - y = accum("days", x, day, st->us_per_day, &leftover_us); + y = accum("days", x, day, CONST_US_PER_DAY(st), &leftover_us); CLEANUP; } if (week) { - y = accum("weeks", x, week, st->us_per_week, &leftover_us); + y = accum("weeks", x, week, CONST_US_PER_WEEK(st), &leftover_us); CLEANUP; } if (leftover_us) { @@ -2679,7 +2825,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) self = microseconds_to_delta_ex(x, type); Py_DECREF(x); + Done: + RELEASE_CURRENT_STATE(st, current_mod); return self; #undef CLEANUP @@ -2792,9 +2940,12 @@ delta_total_seconds(PyObject *self, PyObject *Py_UNUSED(ignored)) if (total_microseconds == NULL) return NULL; - datetime_state *st = get_datetime_state(); - total_seconds = PyNumber_TrueDivide(total_microseconds, st->us_per_second); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); + RELEASE_CURRENT_STATE(st, current_mod); Py_DECREF(total_microseconds); return total_seconds; } @@ -3547,9 +3698,12 @@ date_isocalendar(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) week = 0; } - datetime_state *st = get_datetime_state(); - PyObject *v = iso_calendar_date_new_impl(st->isocalendar_date_type, + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), year, week + 1, day + 1); + RELEASE_CURRENT_STATE(st, current_mod); if (v == NULL) { return NULL; } @@ -4018,9 +4172,8 @@ timezone_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *offset; PyObject *name = NULL; - datetime_state *st = get_datetime_state(); if (PyArg_ParseTupleAndKeywords(args, kw, "O!|U:timezone", timezone_kws, - st->delta_type, &offset, &name)) + DELTA_TYPE(NO_STATE), &offset, &name)) return new_timezone(offset, name); return NULL; @@ -4073,8 +4226,7 @@ timezone_repr(PyDateTime_TimeZone *self) to use Py_TYPE(self)->tp_name here. */ const char *type_name = Py_TYPE(self)->tp_name; - datetime_state *st = get_datetime_state(); - if (((PyObject *)self) == st->utc) { + if ((PyObject *)self == CONST_UTC(NO_STATE)) { return PyUnicode_FromFormat("%s.utc", type_name); } @@ -4096,8 +4248,7 @@ timezone_str(PyDateTime_TimeZone *self) if (self->name != NULL) { return Py_NewRef(self->name); } - datetime_state *st = get_datetime_state(); - if ((PyObject *)self == st->utc || + if ((PyObject *)self == CONST_UTC(NO_STATE) || (GET_TD_DAYS(self->offset) == 0 && GET_TD_SECONDS(self->offset) == 0 && GET_TD_MICROSECONDS(self->offset) == 0)) @@ -4260,7 +4411,7 @@ static PyDateTime_TimeZone * look_up_timezone(PyObject *offset, PyObject *name) { if (offset == utc_timezone.offset && name == NULL) { - return &utc_timezone; + return (PyDateTime_TimeZone *)CONST_UTC(NO_STATE); } return NULL; } @@ -4777,8 +4928,7 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { } PyObject *t; - datetime_state *st = get_datetime_state(); - if ( (PyTypeObject *)cls == st->time_type) { + if ( (PyTypeObject *)cls == TIME_TYPE(NO_STATE)) { t = new_time(hour, minute, second, microsecond, tzinfo, 0); } else { t = PyObject_CallFunction(cls, "iiiiO", @@ -5376,10 +5526,9 @@ datetime_combine(PyObject *cls, PyObject *args, PyObject *kw) PyObject *tzinfo = NULL; PyObject *result = NULL; - datetime_state *st = get_datetime_state(); if (PyArg_ParseTupleAndKeywords(args, kw, "O!O!|O:combine", keywords, - st->date_type, &date, - st->time_type, &time, &tzinfo)) { + DATE_TYPE(NO_STATE), &date, + TIME_TYPE(NO_STATE), &time, &tzinfo)) { if (tzinfo == NULL) { if (HASTZINFO(time)) tzinfo = ((PyDateTime_Time *)time)->tzinfo; @@ -6209,7 +6358,6 @@ local_timezone_from_timestamp(time_t timestamp) delta = new_delta(0, local_time_tm.tm_gmtoff, 0, 1); #else /* HAVE_STRUCT_TM_TM_ZONE */ { - datetime_state *st = get_datetime_state(); PyObject *local_time, *utc_time; struct tm utc_time_tm; char buf[100]; @@ -6264,8 +6412,11 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - datetime_state *st = get_datetime_state(); - delta = datetime_subtract((PyObject *)utc_time, st->epoch); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; @@ -6378,7 +6529,6 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) if (result == NULL) return NULL; - datetime_state *st = get_datetime_state(); /* Make sure result is aware and UTC. */ if (!HASTZINFO(result)) { temp = (PyObject *)result; @@ -6390,7 +6540,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) DATE_GET_MINUTE(result), DATE_GET_SECOND(result), DATE_GET_MICROSECOND(result), - st->utc, + CONST_UTC(NO_STATE), DATE_GET_FOLD(result), Py_TYPE(result)); Py_DECREF(temp); @@ -6399,7 +6549,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) } else { /* Result is already aware - just replace tzinfo. */ - Py_SETREF(result->tzinfo, Py_NewRef(st->utc)); + Py_SETREF(result->tzinfo, Py_NewRef(CONST_UTC(NO_STATE))); } /* Attach new tzinfo and let fromutc() do the rest. */ @@ -6503,9 +6653,12 @@ datetime_timestamp(PyDateTime_DateTime *self, PyObject *Py_UNUSED(ignored)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - datetime_state *st = get_datetime_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + PyObject *delta; - delta = datetime_subtract((PyObject *)self, st->epoch); + delta = datetime_subtract((PyObject *)self, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; result = delta_total_seconds(delta, NULL); @@ -6839,23 +6992,6 @@ get_datetime_capi(void) return &capi; } -static int -datetime_clear(PyObject *module) -{ - datetime_state *st = get_datetime_state(); - - Py_CLEAR(st->us_per_ms); - Py_CLEAR(st->us_per_second); - Py_CLEAR(st->us_per_minute); - Py_CLEAR(st->us_per_hour); - Py_CLEAR(st->us_per_day); - Py_CLEAR(st->us_per_week); - Py_CLEAR(st->seconds_per_day); - Py_CLEAR(st->utc); - Py_CLEAR(st->epoch); - return 0; -} - static PyObject * create_timezone_from_delta(int days, int sec, int ms, int normalize) { @@ -6869,25 +7005,39 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) } static int -init_state(datetime_state *st, PyTypeObject *PyDateTime_IsoCalendarDateType) -{ - // While datetime uses global module "state", we unly initialize it once. - // The PyLong objects created here (once per process) are not decref'd. - if (st->initialized) { +init_state(datetime_state *st, PyObject *module, PyObject *old_module) +{ + /* Each module gets its own heap types. */ +#define ADD_TYPE(FIELD, SPEC, BASE) \ + do { \ + PyObject *cls = PyType_FromModuleAndSpec( \ + module, SPEC, (PyObject *)BASE); \ + if (cls == NULL) { \ + return -1; \ + } \ + st->FIELD = (PyTypeObject *)cls; \ + } while (0) + + ADD_TYPE(isocalendar_date_type, &isocal_spec, &PyTuple_Type); +#undef ADD_TYPE + + if (old_module != NULL) { + assert(old_module != module); + datetime_state *st_old = get_module_state(old_module); + *st = (datetime_state){ + .isocalendar_date_type = st->isocalendar_date_type, + .us_per_ms = Py_NewRef(st_old->us_per_ms), + .us_per_second = Py_NewRef(st_old->us_per_second), + .us_per_minute = Py_NewRef(st_old->us_per_minute), + .us_per_hour = Py_NewRef(st_old->us_per_hour), + .us_per_day = Py_NewRef(st_old->us_per_day), + .us_per_week = Py_NewRef(st_old->us_per_week), + .seconds_per_day = Py_NewRef(st_old->seconds_per_day), + .epoch = Py_NewRef(st_old->epoch), + }; return 0; } - /* Static types exposed by the C-API. */ - st->date_type = &PyDateTime_DateType; - st->datetime_type = &PyDateTime_DateTimeType; - st->delta_type = &PyDateTime_DeltaType; - st->time_type = &PyDateTime_TimeType; - st->tzinfo_type = &PyDateTime_TZInfoType; - st->timezone_type = &PyDateTime_TimeZoneType; - - /* Per-module heap types. */ - st->isocalendar_date_type = PyDateTime_IsoCalendarDateType; - st->us_per_ms = PyLong_FromLong(1000); if (st->us_per_ms == NULL) { return -1; @@ -6921,26 +7071,54 @@ init_state(datetime_state *st, PyTypeObject *PyDateTime_IsoCalendarDateType) return -1; } - /* Init UTC timezone */ - st->utc = create_timezone_from_delta(0, 0, 0, 0); - if (st->utc == NULL) { - return -1; - } - /* Init Unix epoch */ - st->epoch = new_datetime(1970, 1, 1, 0, 0, 0, 0, st->utc, 0); + st->epoch = new_datetime( + 1970, 1, 1, 0, 0, 0, 0, (PyObject *)&utc_timezone, 0); if (st->epoch == NULL) { return -1; } - st->initialized = 1; + return 0; +} + +static int +traverse_state(datetime_state *st, visitproc visit, void *arg) +{ + /* heap types */ + Py_VISIT(st->isocalendar_date_type); return 0; } +static int +clear_state(datetime_state *st) +{ + Py_CLEAR(st->isocalendar_date_type); + Py_CLEAR(st->us_per_ms); + Py_CLEAR(st->us_per_second); + Py_CLEAR(st->us_per_minute); + Py_CLEAR(st->us_per_hour); + Py_CLEAR(st->us_per_day); + Py_CLEAR(st->us_per_week); + Py_CLEAR(st->seconds_per_day); + Py_CLEAR(st->epoch); + return 0; +} + static int _datetime_exec(PyObject *module) { + int rc = -1; + datetime_state *st = get_module_state(module); + + PyInterpreterState *interp = PyInterpreterState_Get(); + PyObject *old_module = get_current_module(interp); + if (PyErr_Occurred()) { + assert(old_module == NULL); + goto error; + } + /* We actually set the "current" module right before a successful return. */ + // `&...` is not a constant expression according to a strict reading // of C standards. Fill tp_base at run-time rather than statically. // See https://bugs.python.org/issue40777 @@ -6953,6 +7131,7 @@ _datetime_exec(PyObject *module) &PyDateTime_TimeType, &PyDateTime_DeltaType, &PyDateTime_TZInfoType, + /* Indirectly, via the utc object. */ &PyDateTime_TimeZoneType, }; @@ -6962,29 +7141,16 @@ _datetime_exec(PyObject *module) } } -#define CREATE_TYPE(VAR, SPEC, BASE) \ - do { \ - VAR = (PyTypeObject *)PyType_FromModuleAndSpec( \ - module, SPEC, (PyObject *)BASE); \ - if (VAR == NULL) { \ - goto error; \ - } \ - } while (0) - - PyTypeObject *PyDateTime_IsoCalendarDateType = NULL; - datetime_state *st = get_datetime_state(); - - if (!st->initialized) { - CREATE_TYPE(PyDateTime_IsoCalendarDateType, &isocal_spec, &PyTuple_Type); - } -#undef CREATE_TYPE - - if (init_state(st, PyDateTime_IsoCalendarDateType) < 0) { + if (init_state(st, module, old_module) < 0) { goto error; } + /* For now we only set the objects on the static types once. + * We will relax that once each types __dict__ is per-interpreter. */ #define DATETIME_ADD_MACRO(dict, c, value_expr) \ do { \ + if (PyDict_GetItemString(dict, c) == NULL) { \ + assert(!PyErr_Occurred()); \ PyObject *value = (value_expr); \ if (value == NULL) { \ goto error; \ @@ -6994,29 +7160,30 @@ _datetime_exec(PyObject *module) goto error; \ } \ Py_DECREF(value); \ + } \ } while(0) /* timedelta values */ - PyObject *d = st->delta_type->tp_dict; + PyObject *d = PyDateTime_DeltaType.tp_dict; DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); DATETIME_ADD_MACRO(d, "max", new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); /* date values */ - d = st->date_type->tp_dict; + d = PyDateTime_DateType.tp_dict; DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); /* time values */ - d = st->time_type->tp_dict; + d = PyDateTime_TimeType.tp_dict; DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); /* datetime values */ - d = st->datetime_type->tp_dict; + d = PyDateTime_DateTimeType.tp_dict; DATETIME_ADD_MACRO(d, "min", new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, @@ -7024,8 +7191,8 @@ _datetime_exec(PyObject *module) DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); /* timezone values */ - d = st->timezone_type->tp_dict; - if (PyDict_SetItemString(d, "utc", st->utc) < 0) { + d = PyDateTime_TimeZoneType.tp_dict; + if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { goto error; } @@ -7034,12 +7201,13 @@ _datetime_exec(PyObject *module) * values. This may change in the future.*/ /* -23:59 */ - PyObject *min = create_timezone_from_delta(-1, 60, 0, 1); - DATETIME_ADD_MACRO(d, "min", min); + DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); /* +23:59 */ - PyObject *max = create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0); - DATETIME_ADD_MACRO(d, "max", max); + DATETIME_ADD_MACRO( + d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); + +#undef DATETIME_ADD_MACRO /* Add module level attributes */ if (PyModule_AddIntMacro(module, MINYEAR) < 0) { @@ -7048,7 +7216,7 @@ _datetime_exec(PyObject *module) if (PyModule_AddIntMacro(module, MAXYEAR) < 0) { goto error; } - if (PyModule_AddObjectRef(module, "UTC", st->utc) < 0) { + if (PyModule_AddObjectRef(module, "UTC", (PyObject *)&utc_timezone) < 0) { goto error; } @@ -7081,13 +7249,20 @@ _datetime_exec(PyObject *module) static_assert(DI100Y == 25 * DI4Y - 1, "DI100Y"); assert(DI100Y == days_before_year(100+1)); - return 0; + if (set_current_module(interp, module) < 0) { + goto error; + } + + rc = 0; + goto finally; error: - datetime_clear(module); - return -1; + clear_state(st); + +finally: + Py_XDECREF(old_module); + return rc; } -#undef DATETIME_ADD_MACRO static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _datetime_exec}, @@ -7096,13 +7271,46 @@ static PyModuleDef_Slot module_slots[] = { {0, NULL}, }; +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + datetime_state *st = get_module_state(mod); + traverse_state(st, visit, arg); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + datetime_state *st = get_module_state(mod); + clear_state(st); + + PyInterpreterState *interp = PyInterpreterState_Get(); + clear_current_module(interp, mod); + + return 0; +} + +static void +module_free(void *mod) +{ + datetime_state *st = get_module_state((PyObject *)mod); + clear_state(st); + + PyInterpreterState *interp = PyInterpreterState_Get(); + clear_current_module(interp, (PyObject *)mod); +} + static PyModuleDef datetimemodule = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_datetime", .m_doc = "Fast implementation of the datetime type.", - .m_size = 0, + .m_size = sizeof(datetime_state), .m_methods = module_methods, .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free, }; PyMODINIT_FUNC From dba7a167dbbd50e83e58df351f3414b7a08e0188 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 3 Jun 2024 18:42:48 -0400 Subject: [PATCH 355/903] gh-117142: Support Importing ctypes in Isolated Interpreters (gh-119991) This makes the support official. Co-authored-by: Kirill Podoprigora --- .../next/Library/2024-06-03-11-18-16.gh-issue-117142.kWTXQo.rst | 2 ++ Modules/_ctypes/_ctypes.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-03-11-18-16.gh-issue-117142.kWTXQo.rst diff --git a/Misc/NEWS.d/next/Library/2024-06-03-11-18-16.gh-issue-117142.kWTXQo.rst b/Misc/NEWS.d/next/Library/2024-06-03-11-18-16.gh-issue-117142.kWTXQo.rst new file mode 100644 index 00000000000000..80734ef3946300 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-03-11-18-16.gh-issue-117142.kWTXQo.rst @@ -0,0 +1,2 @@ +The :mod:`ctypes` module may now be imported in all subinterpreters, including +those that have their own GIL. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 6c1e5f58b95657..1d9534671a4ee8 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5939,7 +5939,7 @@ module_free(void *module) static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _ctypes_mod_exec}, - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; From 105f22ea46ac16866e6df18ebae2a8ba422b7f45 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 3 Jun 2024 19:09:18 -0400 Subject: [PATCH 356/903] gh-117398: Use Per-Interpreter State for the _datetime Static Types (gh-119929) We make use of the same mechanism that we use for the static builtin types. This required a few tweaks. The relevant code could use some cleanup but I opted to avoid the significant churn in this change. I'll tackle that separately. This change is the final piece needed to make _datetime support multiple interpreters. I've updated the module slot accordingly. --- Include/internal/pycore_object.h | 2 +- Include/internal/pycore_typeobject.h | 48 ++- ...-06-01-16-58-43.gh-issue-117398.kR0RW7.rst | 2 + Modules/_datetimemodule.c | 197 +++++++++---- Objects/exceptions.c | 2 +- Objects/object.c | 2 +- Objects/structseq.c | 2 +- Objects/typeobject.c | 278 ++++++++++++------ Objects/unicodeobject.c | 6 +- Objects/weakrefobject.c | 2 +- Python/crossinterp_exceptions.h | 4 +- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 + 13 files changed, 381 insertions(+), 166 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-01-16-58-43.gh-issue-117398.kR0RW7.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index f63e1da6fba025..6f133014ce06e2 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -589,7 +589,7 @@ _PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) if (PyType_Check(op) && ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState( + managed_static_type_state *state = _PyStaticType_GetState( interp, (PyTypeObject *)op); return _PyStaticType_GET_WEAKREFS_LISTPTR(state); } diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 7e533bd138469b..8664ae0e44533f 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -44,10 +44,12 @@ struct type_cache { /* For now we hard-code this to a value for which we are confident all the static builtin types will fit (for all builds). */ -#define _Py_MAX_STATIC_BUILTIN_TYPES 200 +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 +#define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 typedef struct { PyTypeObject *type; + int isbuiltin; int readying; int ready; // XXX tp_dict can probably be statically allocated, @@ -59,7 +61,7 @@ typedef struct { are also some diagnostic uses for the list of weakrefs, so we still keep it. */ PyObject *tp_weaklist; -} static_builtin_state; +} managed_static_type_state; struct types_state { /* Used to set PyTypeObject.tp_version_tag. @@ -105,8 +107,16 @@ struct types_state { num_builtins_initialized is incremented once for each static builtin type. Once initialization is over for a subinterpreter, the value will be the same as for all other interpreters. */ - size_t num_builtins_initialized; - static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES]; + struct { + size_t num_initialized; + managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES]; + } builtins; + /* We apply a similar strategy for managed extension modules. */ + struct { + size_t num_initialized; + size_t next_index; + managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_EXT_TYPES]; + } for_extensions; PyMutex mutex; }; @@ -130,12 +140,35 @@ typedef struct wrapperbase pytype_slotdef; static inline PyObject ** -_PyStaticType_GET_WEAKREFS_LISTPTR(static_builtin_state *state) +_PyStaticType_GET_WEAKREFS_LISTPTR(managed_static_type_state *state) { assert(state != NULL); return &state->tp_weaklist; } +extern int _PyStaticType_InitBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); +extern void _PyStaticType_FiniBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); +extern void _PyStaticType_ClearWeakRefs( + PyInterpreterState *interp, + PyTypeObject *type); +extern managed_static_type_state * _PyStaticType_GetState( + PyInterpreterState *interp, + PyTypeObject *type); + +// Export for '_datetime' shared extension. +PyAPI_FUNC(int) _PyStaticType_InitForExtension( + PyInterpreterState *interp, + PyTypeObject *self); +PyAPI_FUNC(void) _PyStaticType_FiniForExtension( + PyInterpreterState *interp, + PyTypeObject *self, + int final); + + /* Like PyType_GetModuleState, but skips verification * that type is a heap type with an associated module */ static inline void * @@ -151,11 +184,6 @@ _PyType_GetModuleState(PyTypeObject *type) } -extern int _PyStaticType_InitBuiltin(PyInterpreterState *, PyTypeObject *type); -extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTypeObject *); -extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); -extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); - // Export for 'math' shared extension, used via _PyType_IsReady() static inline // function PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); diff --git a/Misc/NEWS.d/next/Library/2024-06-01-16-58-43.gh-issue-117398.kR0RW7.rst b/Misc/NEWS.d/next/Library/2024-06-01-16-58-43.gh-issue-117398.kR0RW7.rst new file mode 100644 index 00000000000000..b0fe06663248f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-01-16-58-43.gh-issue-117398.kR0RW7.rst @@ -0,0 +1,2 @@ +The ``_datetime`` module (C implementation for :mod:`datetime`) now supports +being imported in multiple interpreters. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 16bb4c6980aa08..d6fa273c75e15e 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -111,26 +111,37 @@ get_module_state(PyObject *module) #define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) static PyObject * -get_current_module(PyInterpreterState *interp) +get_current_module(PyInterpreterState *interp, int *p_reloading) { + PyObject *mod = NULL; + int reloading = 0; + PyObject *dict = PyInterpreterState_GetDict(interp); if (dict == NULL) { - return NULL; + goto error; } PyObject *ref = NULL; if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { - return NULL; + goto error; } - if (ref == NULL) { - return NULL; + if (ref != NULL) { + reloading = 1; + if (ref != Py_None) { + (void)PyWeakref_GetRef(ref, &mod); + if (mod == Py_None) { + Py_CLEAR(mod); + } + Py_DECREF(ref); + } } - PyObject *mod = NULL; - (void)PyWeakref_GetRef(ref, &mod); - if (mod == Py_None) { - Py_CLEAR(mod); + if (p_reloading != NULL) { + *p_reloading = reloading; } - Py_DECREF(ref); return mod; + +error: + assert(PyErr_Occurred()); + return NULL; } static PyModuleDef datetimemodule; @@ -139,7 +150,7 @@ static datetime_state * _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *mod = get_current_module(interp); + PyObject *mod = get_current_module(interp, NULL); if (mod == NULL) { assert(!PyErr_Occurred()); if (PyErr_Occurred()) { @@ -184,8 +195,6 @@ clear_current_module(PyInterpreterState *interp, PyObject *expected) { PyObject *exc = PyErr_GetRaisedException(); - PyObject *current = NULL; - PyObject *dict = PyInterpreterState_GetDict(interp); if (dict == NULL) { goto error; @@ -197,7 +206,10 @@ clear_current_module(PyInterpreterState *interp, PyObject *expected) goto error; } if (ref != NULL) { + PyObject *current = NULL; int rc = PyWeakref_GetRef(ref, ¤t); + /* We only need "current" for pointer comparison. */ + Py_XDECREF(current); Py_DECREF(ref); if (rc < 0) { goto error; @@ -208,19 +220,17 @@ clear_current_module(PyInterpreterState *interp, PyObject *expected) } } - if (PyDict_DelItem(dict, INTERP_KEY) < 0) { - if (!PyErr_ExceptionMatches(PyExc_KeyError)) { - goto error; - } + /* We use None to identify that the module was previously loaded. */ + if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) { + goto error; } goto finally; error: - PyErr_Print(); + PyErr_WriteUnraisable(NULL); finally: - Py_XDECREF(current); PyErr_SetRaisedException(exc); } @@ -6947,14 +6957,19 @@ static PyTypeObject PyDateTime_DateTimeType = { }; /* --------------------------------------------------------------------------- - * Module methods and initialization. + * datetime C-API. */ -static PyMethodDef module_methods[] = { - {NULL, NULL} +static PyTypeObject * const capi_types[] = { + &PyDateTime_DateType, + &PyDateTime_DateTimeType, + &PyDateTime_TimeType, + &PyDateTime_DeltaType, + &PyDateTime_TZInfoType, + /* Indirectly, via the utc object. */ + &PyDateTime_TimeZoneType, }; - /* The C-API is process-global. This violates interpreter isolation * due to the objects stored here. Thus each of those objects must * be managed carefully. */ @@ -7004,6 +7019,11 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) return tz; } + +/* --------------------------------------------------------------------------- + * Module state lifecycle. + */ + static int init_state(datetime_state *st, PyObject *module, PyObject *old_module) { @@ -7105,38 +7125,110 @@ clear_state(datetime_state *st) return 0; } + +/* --------------------------------------------------------------------------- + * Global module state. + */ + +// If we make _PyStaticType_*ForExtension() public +// then all this should be managed by the runtime. + +static struct { + PyMutex mutex; + int64_t interp_count; +} _globals = {0}; + +static void +callback_for_interp_exit(void *Py_UNUSED(data)) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + + assert(_globals.interp_count > 0); + PyMutex_Lock(&_globals.mutex); + _globals.interp_count -= 1; + int final = !_globals.interp_count; + PyMutex_Unlock(&_globals.mutex); + + /* They must be done in reverse order so subclasses are finalized + * before base classes. */ + for (size_t i = Py_ARRAY_LENGTH(capi_types); i > 0; i--) { + PyTypeObject *type = capi_types[i-1]; + _PyStaticType_FiniForExtension(interp, type, final); + } +} + +static int +init_static_types(PyInterpreterState *interp, int reloading) +{ + if (reloading) { + return 0; + } + + // `&...` is not a constant expression according to a strict reading + // of C standards. Fill tp_base at run-time rather than statically. + // See https://bugs.python.org/issue40777 + PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; + PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; + + /* Bases classes must be initialized before subclasses, + * so capi_types must have the types in the appropriate order. */ + for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { + PyTypeObject *type = capi_types[i]; + if (_PyStaticType_InitForExtension(interp, type) < 0) { + return -1; + } + } + + PyMutex_Lock(&_globals.mutex); + assert(_globals.interp_count >= 0); + _globals.interp_count += 1; + PyMutex_Unlock(&_globals.mutex); + + /* It could make sense to add a separate callback + * for each of the types. However, for now we can take the simpler + * approach of a single callback. */ + if (PyUnstable_AtExit(interp, callback_for_interp_exit, NULL) < 0) { + callback_for_interp_exit(NULL); + return -1; + } + + return 0; +} + + +/* --------------------------------------------------------------------------- + * Module methods and initialization. + */ + +static PyMethodDef module_methods[] = { + {NULL, NULL} +}; + + static int _datetime_exec(PyObject *module) { int rc = -1; datetime_state *st = get_module_state(module); + int reloading = 0; PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *old_module = get_current_module(interp); + PyObject *old_module = get_current_module(interp, &reloading); if (PyErr_Occurred()) { assert(old_module == NULL); goto error; } /* We actually set the "current" module right before a successful return. */ - // `&...` is not a constant expression according to a strict reading - // of C standards. Fill tp_base at run-time rather than statically. - // See https://bugs.python.org/issue40777 - PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; - PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; - - PyTypeObject *capi_types[] = { - &PyDateTime_DateType, - &PyDateTime_DateTimeType, - &PyDateTime_TimeType, - &PyDateTime_DeltaType, - &PyDateTime_TZInfoType, - /* Indirectly, via the utc object. */ - &PyDateTime_TimeZoneType, - }; + if (init_static_types(interp, reloading) < 0) { + goto error; + } for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { - if (PyModule_AddType(module, capi_types[i]) < 0) { + PyTypeObject *type = capi_types[i]; + const char *name = _PyType_Name(type); + assert(name != NULL); + if (PyModule_AddObjectRef(module, name, (PyObject *)type) < 0) { goto error; } } @@ -7145,11 +7237,8 @@ _datetime_exec(PyObject *module) goto error; } - /* For now we only set the objects on the static types once. - * We will relax that once each types __dict__ is per-interpreter. */ #define DATETIME_ADD_MACRO(dict, c, value_expr) \ do { \ - if (PyDict_GetItemString(dict, c) == NULL) { \ assert(!PyErr_Occurred()); \ PyObject *value = (value_expr); \ if (value == NULL) { \ @@ -7160,30 +7249,29 @@ _datetime_exec(PyObject *module) goto error; \ } \ Py_DECREF(value); \ - } \ } while(0) /* timedelta values */ - PyObject *d = PyDateTime_DeltaType.tp_dict; + PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); DATETIME_ADD_MACRO(d, "max", new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); /* date values */ - d = PyDateTime_DateType.tp_dict; + d = _PyType_GetDict(&PyDateTime_DateType); DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); /* time values */ - d = PyDateTime_TimeType.tp_dict; + d = _PyType_GetDict(&PyDateTime_TimeType); DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); /* datetime values */ - d = PyDateTime_DateTimeType.tp_dict; + d = _PyType_GetDict(&PyDateTime_DateTimeType); DATETIME_ADD_MACRO(d, "min", new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, @@ -7191,7 +7279,7 @@ _datetime_exec(PyObject *module) DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); /* timezone values */ - d = PyDateTime_TimeZoneType.tp_dict; + d = _PyType_GetDict(&PyDateTime_TimeZoneType); if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { goto error; } @@ -7266,7 +7354,7 @@ _datetime_exec(PyObject *module) static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _datetime_exec}, - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -7288,17 +7376,16 @@ module_clear(PyObject *mod) PyInterpreterState *interp = PyInterpreterState_Get(); clear_current_module(interp, mod); + // We take care of the static types via an interpreter atexit hook. + // See callback_for_interp_exit() above. + return 0; } static void module_free(void *mod) { - datetime_state *st = get_module_state((PyObject *)mod); - clear_state(st); - - PyInterpreterState *interp = PyInterpreterState_Get(); - clear_current_module(interp, (PyObject *)mod); + (void)module_clear((PyObject *)mod); } static PyModuleDef datetimemodule = { diff --git a/Objects/exceptions.c b/Objects/exceptions.c index f9cd577c1c16be..3a72cce1dff0c7 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3725,7 +3725,7 @@ _PyExc_FiniTypes(PyInterpreterState *interp) { for (Py_ssize_t i=Py_ARRAY_LENGTH(static_exceptions) - 1; i >= 0; i--) { PyTypeObject *exc = static_exceptions[i].exc; - _PyStaticType_Dealloc(interp, exc); + _PyStaticType_FiniBuiltin(interp, exc); } } diff --git a/Objects/object.c b/Objects/object.c index 5d53e9e5eaba4e..2e9962f4651e1c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2355,7 +2355,7 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types)-1; i>=0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(interp, type); + _PyStaticType_FiniBuiltin(interp, type); } } diff --git a/Objects/structseq.c b/Objects/structseq.c index ec5c5ab45ba813..d8289f2638db0f 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -718,7 +718,7 @@ _PyStructSequence_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) return; } - _PyStaticType_Dealloc(interp, type); + _PyStaticType_FiniBuiltin(interp, type); if (_Py_IsMainInterpreter(interp)) { // Undo _PyStructSequence_InitBuiltinWithFlags(). diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0095a79a2cafec..880ac6b9c009fe 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -131,93 +131,159 @@ type_from_ref(PyObject *ref) #ifndef NDEBUG static inline int -static_builtin_index_is_set(PyTypeObject *self) +managed_static_type_index_is_set(PyTypeObject *self) { return self->tp_subclasses != NULL; } #endif static inline size_t -static_builtin_index_get(PyTypeObject *self) +managed_static_type_index_get(PyTypeObject *self) { - assert(static_builtin_index_is_set(self)); + assert(managed_static_type_index_is_set(self)); /* We store a 1-based index so 0 can mean "not initialized". */ return (size_t)self->tp_subclasses - 1; } static inline void -static_builtin_index_set(PyTypeObject *self, size_t index) +managed_static_type_index_set(PyTypeObject *self, size_t index) { - assert(index < _Py_MAX_STATIC_BUILTIN_TYPES); + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); /* We store a 1-based index so 0 can mean "not initialized". */ self->tp_subclasses = (PyObject *)(index + 1); } static inline void -static_builtin_index_clear(PyTypeObject *self) +managed_static_type_index_clear(PyTypeObject *self) { self->tp_subclasses = NULL; } -static inline static_builtin_state * +static inline managed_static_type_state * static_builtin_state_get(PyInterpreterState *interp, PyTypeObject *self) { - return &(interp->types.builtins[static_builtin_index_get(self)]); + return &(interp->types.builtins.initialized[ + managed_static_type_index_get(self)]); +} + +static inline managed_static_type_state * +static_ext_type_state_get(PyInterpreterState *interp, PyTypeObject *self) +{ + return &(interp->types.for_extensions.initialized[ + managed_static_type_index_get(self)]); +} + +static managed_static_type_state * +managed_static_type_state_get(PyInterpreterState *interp, PyTypeObject *self) +{ + // It's probably a builtin type. + size_t index = managed_static_type_index_get(self); + managed_static_type_state *state = + &(interp->types.builtins.initialized[index]); + if (state->type == self) { + return state; + } + if (index > _Py_MAX_MANAGED_STATIC_EXT_TYPES) { + return state; + } + return &(interp->types.for_extensions.initialized[index]); } /* For static types we store some state in an array on each interpreter. */ -static_builtin_state * +managed_static_type_state * _PyStaticType_GetState(PyInterpreterState *interp, PyTypeObject *self) { assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - return static_builtin_state_get(interp, self); + return managed_static_type_state_get(interp, self); } /* Set the type's per-interpreter state. */ static void -static_builtin_state_init(PyInterpreterState *interp, PyTypeObject *self) +managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int initial) { - if (_Py_IsMainInterpreter(interp)) { - assert(!static_builtin_index_is_set(self)); - static_builtin_index_set(self, interp->types.num_builtins_initialized); + size_t index; + if (initial) { + assert(!managed_static_type_index_is_set(self)); + if (isbuiltin) { + index = interp->types.builtins.num_initialized; + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + } + else { + PyMutex_Lock(&interp->types.mutex); + index = interp->types.for_extensions.next_index; + interp->types.for_extensions.next_index++; + PyMutex_Unlock(&interp->types.mutex); + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + } + managed_static_type_index_set(self, index); } else { - assert(static_builtin_index_get(self) == - interp->types.num_builtins_initialized); + index = managed_static_type_index_get(self); + if (isbuiltin) { + assert(index == interp->types.builtins.num_initialized); + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + } + else { + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + } } - static_builtin_state *state = static_builtin_state_get(interp, self); - /* It should only be called once for each builtin type. */ + managed_static_type_state *state = isbuiltin + ? &(interp->types.builtins.initialized[index]) + : &(interp->types.for_extensions.initialized[index]); + + /* It should only be called once for each builtin type per interpreter. */ assert(state->type == NULL); state->type = self; + state->isbuiltin = isbuiltin; /* state->tp_subclasses is left NULL until init_subclasses() sets it. */ /* state->tp_weaklist is left NULL until insert_head() or insert_after() (in weakrefobject.c) sets it. */ - interp->types.num_builtins_initialized++; + if (isbuiltin) { + interp->types.builtins.num_initialized++; + } + else { + interp->types.for_extensions.num_initialized++; + } } /* Reset the type's per-interpreter state. - This basically undoes what static_builtin_state_init() did. */ + This basically undoes what managed_static_type_state_init() did. */ static void -static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) +managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int final) { - static_builtin_state *state = static_builtin_state_get(interp, self); + managed_static_type_state *state = isbuiltin + ? static_builtin_state_get(interp, self) + : static_ext_type_state_get(interp, self); assert(state->type != NULL); state->type = NULL; assert(state->tp_weaklist == NULL); // It was already cleared out. - if (_Py_IsMainInterpreter(interp)) { - static_builtin_index_clear(self); + if (final) { + managed_static_type_index_clear(self); } - assert(interp->types.num_builtins_initialized > 0); - interp->types.num_builtins_initialized--; + if (isbuiltin) { + assert(interp->types.builtins.num_initialized > 0); + interp->types.builtins.num_initialized--; + } + else { + PyMutex_Lock(&interp->types.mutex); + assert(interp->types.for_extensions.num_initialized > 0); + interp->types.for_extensions.num_initialized--; + if (interp->types.for_extensions.num_initialized == 0) { + interp->types.for_extensions.next_index = 0; + } + PyMutex_Unlock(&interp->types.mutex); + } } -// Also see _PyStaticType_InitBuiltin() and _PyStaticType_Dealloc(). +// Also see _PyStaticType_InitBuiltin() and _PyStaticType_FiniBuiltin(). /* end static builtin helpers */ @@ -227,7 +293,7 @@ start_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); assert(!state->readying); state->readying = 1; @@ -242,7 +308,7 @@ stop_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); assert(state->readying); state->readying = 0; @@ -257,7 +323,7 @@ is_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); return state->readying; } @@ -272,7 +338,7 @@ lookup_tp_dict(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); return state->tp_dict; } @@ -298,7 +364,7 @@ set_tp_dict(PyTypeObject *self, PyObject *dict) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); state->tp_dict = dict; return; @@ -311,7 +377,7 @@ clear_tp_dict(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); Py_CLEAR(state->tp_dict); return; @@ -340,13 +406,13 @@ _PyType_GetBases(PyTypeObject *self) } static inline void -set_tp_bases(PyTypeObject *self, PyObject *bases) +set_tp_bases(PyTypeObject *self, PyObject *bases, int initial) { assert(PyTuple_CheckExact(bases)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { // XXX tp_bases can probably be statically allocated for each // static builtin type. - assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); + assert(initial); assert(self->tp_bases == NULL); if (PyTuple_GET_SIZE(bases) == 0) { assert(self->tp_base == NULL); @@ -363,10 +429,10 @@ set_tp_bases(PyTypeObject *self, PyObject *bases) } static inline void -clear_tp_bases(PyTypeObject *self) +clear_tp_bases(PyTypeObject *self, int final) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (final) { if (self->tp_bases != NULL) { if (PyTuple_GET_SIZE(self->tp_bases) == 0) { Py_CLEAR(self->tp_bases); @@ -413,13 +479,13 @@ _PyType_GetMRO(PyTypeObject *self) } static inline void -set_tp_mro(PyTypeObject *self, PyObject *mro) +set_tp_mro(PyTypeObject *self, PyObject *mro, int initial) { assert(PyTuple_CheckExact(mro)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { // XXX tp_mro can probably be statically allocated for each // static builtin type. - assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); + assert(initial); assert(self->tp_mro == NULL); /* Other checks are done via set_tp_bases. */ _Py_SetImmortal(mro); @@ -428,10 +494,10 @@ set_tp_mro(PyTypeObject *self, PyObject *mro) } static inline void -clear_tp_mro(PyTypeObject *self) +clear_tp_mro(PyTypeObject *self, int final) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (final) { if (self->tp_mro != NULL) { if (PyTuple_GET_SIZE(self->tp_mro) == 0) { Py_CLEAR(self->tp_mro); @@ -457,7 +523,7 @@ init_tp_subclasses(PyTypeObject *self) } if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); state->tp_subclasses = subclasses; return subclasses; } @@ -473,7 +539,7 @@ clear_tp_subclasses(PyTypeObject *self) has no subclass. */ if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); Py_CLEAR(state->tp_subclasses); return; } @@ -485,7 +551,7 @@ lookup_tp_subclasses(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); return state->tp_subclasses; } @@ -774,10 +840,10 @@ _PyTypes_Fini(PyInterpreterState *interp) struct type_cache *cache = &interp->types.type_cache; type_cache_clear(cache, NULL); - assert(interp->types.num_builtins_initialized == 0); + assert(interp->types.builtins.num_initialized == 0); // All the static builtin types should have been finalized already. - for (size_t i = 0; i < _Py_MAX_STATIC_BUILTIN_TYPES; i++) { - assert(interp->types.builtins[i].type == NULL); + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { + assert(interp->types.builtins.initialized[i].type == NULL); } } @@ -1444,7 +1510,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) Py_XDECREF(tuple); if (res < 0) { - set_tp_mro(type, old_mro); + set_tp_mro(type, old_mro, 0); Py_DECREF(new_mro); return -1; } @@ -1545,7 +1611,7 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) assert(old_bases != NULL); PyTypeObject *old_base = type->tp_base; - set_tp_bases(type, Py_NewRef(new_bases)); + set_tp_bases(type, Py_NewRef(new_bases), 0); type->tp_base = (PyTypeObject *)Py_NewRef(new_base); PyObject *temp = PyList_New(0); @@ -1593,7 +1659,7 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) "", 2, 3, &cls, &new_mro, &old_mro); /* Do not rollback if cls has a newer version of MRO. */ if (lookup_tp_mro(cls) == new_mro) { - set_tp_mro(cls, Py_XNewRef(old_mro)); + set_tp_mro(cls, Py_XNewRef(old_mro), 0); Py_DECREF(new_mro); } } @@ -1603,7 +1669,7 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) if (lookup_tp_bases(type) == new_bases) { assert(type->tp_base == new_base); - set_tp_bases(type, old_bases); + set_tp_bases(type, old_bases, 0); type->tp_base = old_base; Py_DECREF(new_bases); @@ -3084,7 +3150,7 @@ mro_invoke(PyTypeObject *type) - Returns -1 in case of an error. */ static int -mro_internal_unlocked(PyTypeObject *type, PyObject **p_old_mro) +mro_internal_unlocked(PyTypeObject *type, int initial, PyObject **p_old_mro) { ASSERT_TYPE_LOCK_HELD(); @@ -3107,7 +3173,7 @@ mro_internal_unlocked(PyTypeObject *type, PyObject **p_old_mro) return 0; } - set_tp_mro(type, new_mro); + set_tp_mro(type, new_mro, initial); type_mro_modified(type, new_mro); /* corner case: the super class might have been hidden @@ -3137,7 +3203,7 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) { int res; BEGIN_TYPE_LOCK() - res = mro_internal_unlocked(type, p_old_mro); + res = mro_internal_unlocked(type, 0, p_old_mro); END_TYPE_LOCK() return res; } @@ -3732,7 +3798,7 @@ type_new_alloc(type_new_ctx *ctx) type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; - set_tp_bases(type, Py_NewRef(ctx->bases)); + set_tp_bases(type, Py_NewRef(ctx->bases), 1); type->tp_base = (PyTypeObject *)Py_NewRef(ctx->base); type->tp_dealloc = subtype_dealloc; @@ -4722,7 +4788,7 @@ _PyType_FromMetaclass_impl( /* Set slots we have prepared */ type->tp_base = (PyTypeObject *)Py_NewRef(base); - set_tp_bases(type, bases); + set_tp_bases(type, bases, 1); bases = NULL; // We give our reference to bases to the type type->tp_doc = tp_doc; @@ -5627,7 +5693,7 @@ type_dealloc_common(PyTypeObject *type) static void -clear_static_tp_subclasses(PyTypeObject *type) +clear_static_tp_subclasses(PyTypeObject *type, int isbuiltin) { PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses == NULL) { @@ -5664,47 +5730,64 @@ clear_static_tp_subclasses(PyTypeObject *type) continue; } // All static builtin subtypes should have been finalized already. - assert(!(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + assert(!isbuiltin || !(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); Py_DECREF(subclass); } +#else + (void)isbuiltin; #endif clear_tp_subclasses(type); } static void -clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) +clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type, + int isbuiltin, int final) { - if (_Py_IsMainInterpreter(interp)) { + if (final) { Py_CLEAR(type->tp_cache); } clear_tp_dict(type); - clear_tp_bases(type); - clear_tp_mro(type); - clear_static_tp_subclasses(type); + clear_tp_bases(type, final); + clear_tp_mro(type, final); + clear_static_tp_subclasses(type, isbuiltin); } -void -_PyStaticType_Dealloc(PyInterpreterState *interp, PyTypeObject *type) + +static void +fini_static_type(PyInterpreterState *interp, PyTypeObject *type, + int isbuiltin, int final) { assert(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); assert(_Py_IsImmortal((PyObject *)type)); type_dealloc_common(type); - clear_static_type_objects(interp, type); + clear_static_type_objects(interp, type, isbuiltin, final); - if (_Py_IsMainInterpreter(interp)) { + if (final) { type->tp_flags &= ~Py_TPFLAGS_READY; type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; type->tp_version_tag = 0; } _PyStaticType_ClearWeakRefs(interp, type); - static_builtin_state_clear(interp, type); + managed_static_type_state_clear(interp, type, isbuiltin, final); /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ } +void +_PyStaticType_FiniForExtension(PyInterpreterState *interp, PyTypeObject *type, int final) +{ + fini_static_type(interp, type, 0, final); +} + +void +_PyStaticType_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) +{ + fini_static_type(interp, type, 1, _Py_IsMainInterpreter(interp)); +} + static void type_dealloc(PyObject *self) @@ -7758,10 +7841,10 @@ type_ready_set_type(PyTypeObject *type) } static int -type_ready_set_bases(PyTypeObject *type) +type_ready_set_bases(PyTypeObject *type, int initial) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (!_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (!initial) { assert(lookup_tp_bases(type) != NULL); return 0; } @@ -7780,7 +7863,7 @@ type_ready_set_bases(PyTypeObject *type) if (bases == NULL) { return -1; } - set_tp_bases(type, bases); + set_tp_bases(type, bases, 1); } return 0; } @@ -7890,12 +7973,12 @@ type_ready_preheader(PyTypeObject *type) } static int -type_ready_mro(PyTypeObject *type) +type_ready_mro(PyTypeObject *type, int initial) { ASSERT_TYPE_LOCK_HELD(); if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (!_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (!initial) { assert(lookup_tp_mro(type) != NULL); return 0; } @@ -7903,7 +7986,7 @@ type_ready_mro(PyTypeObject *type) } /* Calculate method resolution order */ - if (mro_internal_unlocked(type, NULL) < 0) { + if (mro_internal_unlocked(type, initial, NULL) < 0) { return -1; } PyObject *mro = lookup_tp_mro(type); @@ -8058,7 +8141,7 @@ type_ready_add_subclasses(PyTypeObject *type) // Set tp_new and the "__new__" key in the type dictionary. // Use the Py_TPFLAGS_DISALLOW_INSTANTIATION flag. static int -type_ready_set_new(PyTypeObject *type, int rerunbuiltin) +type_ready_set_new(PyTypeObject *type, int initial) { PyTypeObject *base = type->tp_base; /* The condition below could use some explanation. @@ -8080,7 +8163,7 @@ type_ready_set_new(PyTypeObject *type, int rerunbuiltin) if (!(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION)) { if (type->tp_new != NULL) { - if (!rerunbuiltin || base == NULL || type->tp_new != base->tp_new) { + if (initial || base == NULL || type->tp_new != base->tp_new) { // If "__new__" key does not exists in the type dictionary, // set it to tp_new_wrapper(). if (add_tp_new_wrapper(type) < 0) { @@ -8162,7 +8245,7 @@ type_ready_post_checks(PyTypeObject *type) static int -type_ready(PyTypeObject *type, int rerunbuiltin) +type_ready(PyTypeObject *type, int initial) { ASSERT_TYPE_LOCK_HELD(); @@ -8192,19 +8275,19 @@ type_ready(PyTypeObject *type, int rerunbuiltin) if (type_ready_set_type(type) < 0) { goto error; } - if (type_ready_set_bases(type) < 0) { + if (type_ready_set_bases(type, initial) < 0) { goto error; } - if (type_ready_mro(type) < 0) { + if (type_ready_mro(type, initial) < 0) { goto error; } - if (type_ready_set_new(type, rerunbuiltin) < 0) { + if (type_ready_set_new(type, initial) < 0) { goto error; } if (type_ready_fill_dict(type) < 0) { goto error; } - if (!rerunbuiltin) { + if (initial) { if (type_ready_inherit(type) < 0) { goto error; } @@ -8218,7 +8301,7 @@ type_ready(PyTypeObject *type, int rerunbuiltin) if (type_ready_add_subclasses(type) < 0) { goto error; } - if (!rerunbuiltin) { + if (initial) { if (type_ready_managed_dict(type) < 0) { goto error; } @@ -8258,7 +8341,7 @@ PyType_Ready(PyTypeObject *type) int res; BEGIN_TYPE_LOCK() if (!(type->tp_flags & Py_TPFLAGS_READY)) { - res = type_ready(type, 0); + res = type_ready(type, 1); } else { res = 0; assert(_PyType_CheckConsistency(type)); @@ -8267,17 +8350,18 @@ PyType_Ready(PyTypeObject *type) return res; } -int -_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) + +static int +init_static_type(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int initial) { assert(_Py_IsImmortal((PyObject *)self)); assert(!(self->tp_flags & Py_TPFLAGS_HEAPTYPE)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); - int ismain = _Py_IsMainInterpreter(interp); if ((self->tp_flags & Py_TPFLAGS_READY) == 0) { - assert(ismain); + assert(initial); self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; @@ -8287,24 +8371,36 @@ _PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; } else { - assert(!ismain); + assert(!initial); assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); assert(self->tp_flags & Py_TPFLAGS_VALID_VERSION_TAG); } - static_builtin_state_init(interp, self); + managed_static_type_state_init(interp, self, isbuiltin, initial); int res; BEGIN_TYPE_LOCK(); - res = type_ready(self, !ismain); + res = type_ready(self, initial); END_TYPE_LOCK() if (res < 0) { _PyStaticType_ClearWeakRefs(interp, self); - static_builtin_state_clear(interp, self); + managed_static_type_state_clear(interp, self, isbuiltin, initial); } return res; } +int +_PyStaticType_InitForExtension(PyInterpreterState *interp, PyTypeObject *self) +{ + return init_static_type(interp, self, 0, ((self->tp_flags & Py_TPFLAGS_READY) == 0)); +} + +int +_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) +{ + return init_static_type(interp, self, 1, _Py_IsMainInterpreter(interp)); +} + static int add_subclass(PyTypeObject *base, PyTypeObject *type) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 53160f1799f2cc..3b0b4173408724 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15522,9 +15522,9 @@ unicode_is_finalizing(void) void _PyUnicode_FiniTypes(PyInterpreterState *interp) { - _PyStaticType_Dealloc(interp, &EncodingMapType); - _PyStaticType_Dealloc(interp, &PyFieldNameIter_Type); - _PyStaticType_Dealloc(interp, &PyFormatterIter_Type); + _PyStaticType_FiniBuiltin(interp, &EncodingMapType); + _PyStaticType_FiniBuiltin(interp, &PyFieldNameIter_Type); + _PyStaticType_FiniBuiltin(interp, &PyFormatterIter_Type); } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 88afaec86827ed..3b027e1b518ba6 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1066,7 +1066,7 @@ PyObject_ClearWeakRefs(PyObject *object) void _PyStaticType_ClearWeakRefs(PyInterpreterState *interp, PyTypeObject *type) { - static_builtin_state *state = _PyStaticType_GetState(interp, type); + managed_static_type_state *state = _PyStaticType_GetState(interp, type); PyObject **list = _PyStaticType_GET_WEAKREFS_LISTPTR(state); // This is safe to do without holding the lock in free-threaded builds; // there is only one thread running and no new threads can be created. diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h index 6ecc10c7955fd8..278511da615c75 100644 --- a/Python/crossinterp_exceptions.h +++ b/Python/crossinterp_exceptions.h @@ -90,6 +90,6 @@ static void fini_exceptions(PyInterpreterState *interp) { // Likewise with _fini_not_shareable_error_type(). - _PyStaticType_Dealloc(interp, &_PyExc_InterpreterNotFoundError); - _PyStaticType_Dealloc(interp, &_PyExc_InterpreterError); + _PyStaticType_FiniBuiltin(interp, &_PyExc_InterpreterNotFoundError); + _PyStaticType_FiniBuiltin(interp, &_PyExc_InterpreterError); } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 711ae343a8d876..4586a59f6ac2ef 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -307,6 +307,7 @@ Python/crossinterp_exceptions.h - PyExc_InterpreterNotFoundError - Modules/_datetimemodule.c - zero_delta - Modules/_datetimemodule.c - utc_timezone - Modules/_datetimemodule.c - capi - +Modules/_datetimemodule.c - _globals - Objects/boolobject.c - _Py_FalseStruct - Objects/boolobject.c - _Py_TrueStruct - Objects/dictobject.c - empty_keys_struct - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index a3bdf0396fd3e1..466f25daa14dc6 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -217,6 +217,7 @@ Modules/_datetimemodule.c - max_fold_seconds - Modules/_datetimemodule.c datetime_isoformat specs - Modules/_datetimemodule.c parse_hh_mm_ss_ff correction - Modules/_datetimemodule.c time_isoformat specs - +Modules/_datetimemodule.c - capi_types - Modules/_decimal/_decimal.c - cond_map_template - Modules/_decimal/_decimal.c - dec_signal_string - Modules/_decimal/_decimal.c - dflt_ctx - From 31a4fb3c74a0284436343858803b54471e2dc9c7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 4 Jun 2024 03:10:15 +0200 Subject: [PATCH 357/903] gh-119724: Revert "bpo-45759: Better error messages for non-matching 'elif'/'else' statements (#29513)" (#119974) This reverts commit 1c8f912ebdfdb146cd7dd2d7a3a67d2c5045ddb0. --- Grammar/python.gram | 5 - Lib/test/test_syntax.py | 61 +- ...-06-03-13-25-04.gh-issue-119724.EH1dkA.rst | 3 + Parser/parser.c | 998 ++++++++---------- 4 files changed, 455 insertions(+), 612 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-03-13-25-04.gh-issue-119724.EH1dkA.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index 1734479276dd5b..b14e5dd096cdf4 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -127,7 +127,6 @@ simple_stmt[stmt_ty] (memo): | &'nonlocal' nonlocal_stmt compound_stmt[stmt_ty]: - | invalid_compound_stmt | &('def' | '@' | 'async') function_def | &'if' if_stmt | &('class' | '@') class_def @@ -1323,10 +1322,6 @@ invalid_import_from_targets: | token=NEWLINE { RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } -invalid_compound_stmt: - | a='elif' named_expression ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "'elif' must match an if-statement here") } - | a='else' ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "'else' must match a valid statement here") } - invalid_with_stmt: | ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | ['async'] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index b978838ea7003f..491f7fd7908e97 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1853,28 +1853,6 @@ Traceback (most recent call last): SyntaxError: positional patterns follow keyword patterns -Non-matching 'elif'/'else' statements: - - >>> if a == b: - ... ... - ... elif a == c: - Traceback (most recent call last): - SyntaxError: 'elif' must match an if-statement here - - >>> if x == y: - ... ... - ... else: - Traceback (most recent call last): - SyntaxError: 'else' must match a valid statement here - - >>> elif m == n: - Traceback (most recent call last): - SyntaxError: 'elif' must match an if-statement here - - >>> else: - Traceback (most recent call last): - SyntaxError: 'else' must match a valid statement here - Uses of the star operator which should fail: A[:*b] @@ -2167,8 +2145,8 @@ def _check_error(self, code, errtext, lineno=None, offset=None, end_lineno=None, end_offset=None): """Check that compiling code raises SyntaxError with errtext. - errtext is a regular expression that must be present in the - test of the exception raised. If subclass is specified, it + errtest is a regular expression that must be present in the + test of the exception raised. If subclass is specified it is the expected subclass of SyntaxError (e.g. IndentationError). """ try: @@ -2192,22 +2170,6 @@ def _check_error(self, code, errtext, else: self.fail("compile() did not raise SyntaxError") - def _check_noerror(self, code, - errtext="compile() raised unexpected SyntaxError", - filename="", mode="exec", subclass=None): - """Check that compiling code does not raise a SyntaxError. - - errtext is the message passed to self.fail if there is - a SyntaxError. If the subclass parameter is specified, - it is the subclass of SyntaxError (e.g. IndentationError) - that the raised error is checked against. - """ - try: - compile(code, filename, mode) - except SyntaxError as err: - if (not subclass) or isinstance(err, subclass): - self.fail(errtext) - def test_expression_with_assignment(self): self._check_error( "print(end1 + end2 = ' ')", @@ -2609,25 +2571,6 @@ def test_syntax_error_on_deeply_nested_blocks(self): """ self._check_error(source, "too many statically nested blocks") - def test_syntax_error_non_matching_elif_else_statements(self): - # Check bpo-45759: 'elif' statements that doesn't match an - # if-statement or 'else' statements that doesn't match any - # valid else-able statement (e.g. 'while') - self._check_error( - "elif m == n:\n ...", - "'elif' must match an if-statement here") - self._check_error( - "else:\n ...", - "'else' must match a valid statement here") - self._check_noerror("if a == b:\n ...\nelif a == c:\n ...") - self._check_noerror("if x == y:\n ...\nelse:\n ...") - self._check_error( - "else = 123", - "invalid syntax") - self._check_error( - "elif 55 = 123", - "cannot assign to literal here") - @support.cpython_only def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-03-13-25-04.gh-issue-119724.EH1dkA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-03-13-25-04.gh-issue-119724.EH1dkA.rst new file mode 100644 index 00000000000000..78dc48da934cf6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-03-13-25-04.gh-issue-119724.EH1dkA.rst @@ -0,0 +1,3 @@ +Reverted improvements to error messages for ``elif``/``else`` statements not +matching any valid statements, which made in hard to locate the syntax +errors inside those ``elif``/``else`` blocks. diff --git a/Parser/parser.c b/Parser/parser.c index fec3353d30cb4d..05cd93c2c92f29 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -21,28 +21,28 @@ static KeywordToken *reserved_keywords[] = { (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) { - {"if", 662}, - {"as", 660}, - {"in", 673}, + {"if", 660}, + {"as", 658}, + {"in", 671}, {"or", 581}, {"is", 589}, {NULL, -1}, }, (KeywordToken[]) { {"del", 613}, - {"def", 677}, - {"for", 672}, - {"try", 644}, + {"def", 675}, + {"for", 670}, + {"try", 642}, {"and", 582}, - {"not", 681}, + {"not", 679}, {NULL, -1}, }, (KeywordToken[]) { {"from", 621}, {"pass", 504}, - {"with", 635}, - {"elif", 664}, - {"else", 665}, + {"with", 633}, + {"elif", 662}, + {"else", 663}, {"None", 611}, {"True", 610}, {NULL, -1}, @@ -51,9 +51,9 @@ static KeywordToken *reserved_keywords[] = { {"raise", 525}, {"yield", 580}, {"break", 508}, - {"async", 676}, - {"class", 679}, - {"while", 667}, + {"async", 674}, + {"class", 677}, + {"while", 665}, {"False", 612}, {"await", 590}, {NULL, -1}, @@ -63,12 +63,12 @@ static KeywordToken *reserved_keywords[] = { {"import", 622}, {"assert", 529}, {"global", 526}, - {"except", 657}, + {"except", 655}, {"lambda", 609}, {NULL, -1}, }, (KeywordToken[]) { - {"finally", 653}, + {"finally", 651}, {NULL, -1}, }, (KeywordToken[]) { @@ -308,316 +308,315 @@ static char *soft_keywords[] = { #define invalid_group_type 1221 #define invalid_import_type 1222 #define invalid_import_from_targets_type 1223 -#define invalid_compound_stmt_type 1224 -#define invalid_with_stmt_type 1225 -#define invalid_with_stmt_indent_type 1226 -#define invalid_try_stmt_type 1227 -#define invalid_except_stmt_type 1228 -#define invalid_finally_stmt_type 1229 -#define invalid_except_stmt_indent_type 1230 -#define invalid_except_star_stmt_indent_type 1231 -#define invalid_match_stmt_type 1232 -#define invalid_case_block_type 1233 -#define invalid_as_pattern_type 1234 -#define invalid_class_pattern_type 1235 -#define invalid_class_argument_pattern_type 1236 -#define invalid_if_stmt_type 1237 -#define invalid_elif_stmt_type 1238 -#define invalid_else_stmt_type 1239 -#define invalid_while_stmt_type 1240 -#define invalid_for_stmt_type 1241 -#define invalid_def_raw_type 1242 -#define invalid_class_def_raw_type 1243 -#define invalid_double_starred_kvpairs_type 1244 -#define invalid_kvpair_type 1245 -#define invalid_starred_expression_unpacking_type 1246 -#define invalid_starred_expression_type 1247 -#define invalid_replacement_field_type 1248 -#define invalid_conversion_character_type 1249 -#define invalid_arithmetic_type 1250 -#define invalid_factor_type 1251 -#define invalid_type_params_type 1252 -#define _loop0_1_type 1253 -#define _loop0_2_type 1254 -#define _loop1_3_type 1255 -#define _loop0_5_type 1256 -#define _gather_4_type 1257 -#define _tmp_6_type 1258 -#define _tmp_7_type 1259 -#define _tmp_8_type 1260 -#define _tmp_9_type 1261 -#define _tmp_10_type 1262 -#define _tmp_11_type 1263 -#define _tmp_12_type 1264 -#define _tmp_13_type 1265 -#define _loop1_14_type 1266 -#define _tmp_15_type 1267 -#define _tmp_16_type 1268 -#define _tmp_17_type 1269 -#define _loop0_19_type 1270 -#define _gather_18_type 1271 -#define _loop0_21_type 1272 -#define _gather_20_type 1273 -#define _tmp_22_type 1274 -#define _tmp_23_type 1275 -#define _loop0_24_type 1276 -#define _loop1_25_type 1277 -#define _loop0_27_type 1278 -#define _gather_26_type 1279 -#define _tmp_28_type 1280 -#define _loop0_30_type 1281 -#define _gather_29_type 1282 -#define _tmp_31_type 1283 -#define _loop1_32_type 1284 -#define _tmp_33_type 1285 -#define _tmp_34_type 1286 -#define _tmp_35_type 1287 -#define _loop0_36_type 1288 -#define _loop0_37_type 1289 -#define _loop0_38_type 1290 -#define _loop1_39_type 1291 -#define _loop0_40_type 1292 -#define _loop1_41_type 1293 -#define _loop1_42_type 1294 -#define _loop1_43_type 1295 -#define _loop0_44_type 1296 -#define _loop1_45_type 1297 -#define _loop0_46_type 1298 -#define _loop1_47_type 1299 -#define _loop0_48_type 1300 -#define _loop0_49_type 1301 -#define _loop1_50_type 1302 -#define _loop0_52_type 1303 -#define _gather_51_type 1304 -#define _loop0_54_type 1305 -#define _gather_53_type 1306 -#define _loop0_56_type 1307 -#define _gather_55_type 1308 -#define _loop0_58_type 1309 -#define _gather_57_type 1310 -#define _tmp_59_type 1311 -#define _loop1_60_type 1312 -#define _loop1_61_type 1313 -#define _tmp_62_type 1314 -#define _tmp_63_type 1315 -#define _loop1_64_type 1316 -#define _loop0_66_type 1317 -#define _gather_65_type 1318 -#define _tmp_67_type 1319 -#define _tmp_68_type 1320 -#define _tmp_69_type 1321 -#define _tmp_70_type 1322 -#define _loop0_72_type 1323 -#define _gather_71_type 1324 -#define _loop0_74_type 1325 -#define _gather_73_type 1326 -#define _tmp_75_type 1327 -#define _loop0_77_type 1328 -#define _gather_76_type 1329 -#define _loop0_79_type 1330 -#define _gather_78_type 1331 -#define _loop0_81_type 1332 -#define _gather_80_type 1333 -#define _loop1_82_type 1334 -#define _loop1_83_type 1335 -#define _loop0_85_type 1336 -#define _gather_84_type 1337 -#define _loop1_86_type 1338 -#define _loop1_87_type 1339 -#define _loop1_88_type 1340 -#define _tmp_89_type 1341 -#define _loop0_91_type 1342 -#define _gather_90_type 1343 -#define _tmp_92_type 1344 -#define _tmp_93_type 1345 -#define _tmp_94_type 1346 -#define _tmp_95_type 1347 -#define _tmp_96_type 1348 -#define _tmp_97_type 1349 -#define _loop0_98_type 1350 -#define _loop0_99_type 1351 -#define _loop0_100_type 1352 -#define _loop1_101_type 1353 -#define _loop0_102_type 1354 -#define _loop1_103_type 1355 -#define _loop1_104_type 1356 -#define _loop1_105_type 1357 -#define _loop0_106_type 1358 -#define _loop1_107_type 1359 -#define _loop0_108_type 1360 -#define _loop1_109_type 1361 -#define _loop0_110_type 1362 -#define _loop1_111_type 1363 -#define _loop0_112_type 1364 -#define _loop0_113_type 1365 -#define _loop1_114_type 1366 -#define _tmp_115_type 1367 -#define _loop0_117_type 1368 -#define _gather_116_type 1369 -#define _loop1_118_type 1370 -#define _loop0_119_type 1371 -#define _loop0_120_type 1372 -#define _tmp_121_type 1373 -#define _loop0_123_type 1374 -#define _gather_122_type 1375 -#define _tmp_124_type 1376 -#define _loop0_126_type 1377 -#define _gather_125_type 1378 -#define _loop0_128_type 1379 -#define _gather_127_type 1380 -#define _loop0_130_type 1381 -#define _gather_129_type 1382 -#define _loop0_132_type 1383 -#define _gather_131_type 1384 -#define _loop0_133_type 1385 -#define _loop0_135_type 1386 -#define _gather_134_type 1387 -#define _loop1_136_type 1388 -#define _tmp_137_type 1389 -#define _loop0_139_type 1390 -#define _gather_138_type 1391 -#define _loop0_141_type 1392 -#define _gather_140_type 1393 -#define _loop0_143_type 1394 -#define _gather_142_type 1395 -#define _loop0_145_type 1396 -#define _gather_144_type 1397 -#define _loop0_147_type 1398 -#define _gather_146_type 1399 -#define _tmp_148_type 1400 -#define _tmp_149_type 1401 -#define _loop0_151_type 1402 -#define _gather_150_type 1403 -#define _tmp_152_type 1404 -#define _tmp_153_type 1405 -#define _tmp_154_type 1406 -#define _tmp_155_type 1407 -#define _tmp_156_type 1408 -#define _tmp_157_type 1409 -#define _tmp_158_type 1410 -#define _tmp_159_type 1411 -#define _tmp_160_type 1412 -#define _tmp_161_type 1413 -#define _loop0_162_type 1414 -#define _loop0_163_type 1415 -#define _loop0_164_type 1416 -#define _tmp_165_type 1417 -#define _tmp_166_type 1418 -#define _tmp_167_type 1419 -#define _tmp_168_type 1420 -#define _loop0_169_type 1421 -#define _loop0_170_type 1422 -#define _loop0_171_type 1423 -#define _loop1_172_type 1424 -#define _tmp_173_type 1425 -#define _loop0_174_type 1426 -#define _tmp_175_type 1427 -#define _loop0_176_type 1428 -#define _loop1_177_type 1429 -#define _tmp_178_type 1430 -#define _tmp_179_type 1431 -#define _tmp_180_type 1432 -#define _loop0_181_type 1433 -#define _tmp_182_type 1434 -#define _tmp_183_type 1435 -#define _loop1_184_type 1436 -#define _tmp_185_type 1437 -#define _loop0_186_type 1438 -#define _loop0_187_type 1439 -#define _loop0_188_type 1440 -#define _loop0_190_type 1441 -#define _gather_189_type 1442 -#define _tmp_191_type 1443 -#define _loop0_192_type 1444 -#define _tmp_193_type 1445 -#define _loop0_194_type 1446 -#define _loop1_195_type 1447 -#define _loop1_196_type 1448 -#define _tmp_197_type 1449 -#define _tmp_198_type 1450 -#define _loop0_199_type 1451 -#define _tmp_200_type 1452 -#define _tmp_201_type 1453 -#define _tmp_202_type 1454 -#define _tmp_203_type 1455 -#define _loop0_205_type 1456 -#define _gather_204_type 1457 -#define _loop0_207_type 1458 -#define _gather_206_type 1459 -#define _loop0_209_type 1460 -#define _gather_208_type 1461 -#define _loop0_211_type 1462 -#define _gather_210_type 1463 -#define _loop0_213_type 1464 -#define _gather_212_type 1465 -#define _tmp_214_type 1466 -#define _loop0_215_type 1467 -#define _loop1_216_type 1468 -#define _tmp_217_type 1469 -#define _loop0_218_type 1470 -#define _loop1_219_type 1471 -#define _tmp_220_type 1472 -#define _tmp_221_type 1473 -#define _tmp_222_type 1474 -#define _tmp_223_type 1475 -#define _tmp_224_type 1476 -#define _tmp_225_type 1477 -#define _tmp_226_type 1478 -#define _tmp_227_type 1479 -#define _tmp_228_type 1480 -#define _tmp_229_type 1481 -#define _tmp_230_type 1482 -#define _loop0_232_type 1483 -#define _gather_231_type 1484 -#define _tmp_233_type 1485 -#define _tmp_234_type 1486 -#define _tmp_235_type 1487 -#define _tmp_236_type 1488 -#define _tmp_237_type 1489 -#define _tmp_238_type 1490 -#define _tmp_239_type 1491 -#define _loop0_240_type 1492 -#define _tmp_241_type 1493 -#define _tmp_242_type 1494 -#define _tmp_243_type 1495 -#define _tmp_244_type 1496 -#define _tmp_245_type 1497 -#define _tmp_246_type 1498 -#define _tmp_247_type 1499 -#define _tmp_248_type 1500 -#define _tmp_249_type 1501 -#define _tmp_250_type 1502 -#define _tmp_251_type 1503 -#define _tmp_252_type 1504 -#define _tmp_253_type 1505 -#define _tmp_254_type 1506 -#define _tmp_255_type 1507 -#define _tmp_256_type 1508 -#define _tmp_257_type 1509 -#define _tmp_258_type 1510 -#define _tmp_259_type 1511 -#define _tmp_260_type 1512 -#define _tmp_261_type 1513 -#define _tmp_262_type 1514 -#define _tmp_263_type 1515 -#define _tmp_264_type 1516 -#define _tmp_265_type 1517 -#define _loop0_266_type 1518 -#define _tmp_267_type 1519 -#define _tmp_268_type 1520 -#define _tmp_269_type 1521 -#define _tmp_270_type 1522 -#define _tmp_271_type 1523 -#define _tmp_272_type 1524 -#define _loop0_274_type 1525 -#define _gather_273_type 1526 -#define _tmp_275_type 1527 -#define _tmp_276_type 1528 -#define _tmp_277_type 1529 -#define _tmp_278_type 1530 -#define _tmp_279_type 1531 -#define _tmp_280_type 1532 -#define _tmp_281_type 1533 +#define invalid_with_stmt_type 1224 +#define invalid_with_stmt_indent_type 1225 +#define invalid_try_stmt_type 1226 +#define invalid_except_stmt_type 1227 +#define invalid_finally_stmt_type 1228 +#define invalid_except_stmt_indent_type 1229 +#define invalid_except_star_stmt_indent_type 1230 +#define invalid_match_stmt_type 1231 +#define invalid_case_block_type 1232 +#define invalid_as_pattern_type 1233 +#define invalid_class_pattern_type 1234 +#define invalid_class_argument_pattern_type 1235 +#define invalid_if_stmt_type 1236 +#define invalid_elif_stmt_type 1237 +#define invalid_else_stmt_type 1238 +#define invalid_while_stmt_type 1239 +#define invalid_for_stmt_type 1240 +#define invalid_def_raw_type 1241 +#define invalid_class_def_raw_type 1242 +#define invalid_double_starred_kvpairs_type 1243 +#define invalid_kvpair_type 1244 +#define invalid_starred_expression_unpacking_type 1245 +#define invalid_starred_expression_type 1246 +#define invalid_replacement_field_type 1247 +#define invalid_conversion_character_type 1248 +#define invalid_arithmetic_type 1249 +#define invalid_factor_type 1250 +#define invalid_type_params_type 1251 +#define _loop0_1_type 1252 +#define _loop0_2_type 1253 +#define _loop1_3_type 1254 +#define _loop0_5_type 1255 +#define _gather_4_type 1256 +#define _tmp_6_type 1257 +#define _tmp_7_type 1258 +#define _tmp_8_type 1259 +#define _tmp_9_type 1260 +#define _tmp_10_type 1261 +#define _tmp_11_type 1262 +#define _tmp_12_type 1263 +#define _tmp_13_type 1264 +#define _loop1_14_type 1265 +#define _tmp_15_type 1266 +#define _tmp_16_type 1267 +#define _tmp_17_type 1268 +#define _loop0_19_type 1269 +#define _gather_18_type 1270 +#define _loop0_21_type 1271 +#define _gather_20_type 1272 +#define _tmp_22_type 1273 +#define _tmp_23_type 1274 +#define _loop0_24_type 1275 +#define _loop1_25_type 1276 +#define _loop0_27_type 1277 +#define _gather_26_type 1278 +#define _tmp_28_type 1279 +#define _loop0_30_type 1280 +#define _gather_29_type 1281 +#define _tmp_31_type 1282 +#define _loop1_32_type 1283 +#define _tmp_33_type 1284 +#define _tmp_34_type 1285 +#define _tmp_35_type 1286 +#define _loop0_36_type 1287 +#define _loop0_37_type 1288 +#define _loop0_38_type 1289 +#define _loop1_39_type 1290 +#define _loop0_40_type 1291 +#define _loop1_41_type 1292 +#define _loop1_42_type 1293 +#define _loop1_43_type 1294 +#define _loop0_44_type 1295 +#define _loop1_45_type 1296 +#define _loop0_46_type 1297 +#define _loop1_47_type 1298 +#define _loop0_48_type 1299 +#define _loop0_49_type 1300 +#define _loop1_50_type 1301 +#define _loop0_52_type 1302 +#define _gather_51_type 1303 +#define _loop0_54_type 1304 +#define _gather_53_type 1305 +#define _loop0_56_type 1306 +#define _gather_55_type 1307 +#define _loop0_58_type 1308 +#define _gather_57_type 1309 +#define _tmp_59_type 1310 +#define _loop1_60_type 1311 +#define _loop1_61_type 1312 +#define _tmp_62_type 1313 +#define _tmp_63_type 1314 +#define _loop1_64_type 1315 +#define _loop0_66_type 1316 +#define _gather_65_type 1317 +#define _tmp_67_type 1318 +#define _tmp_68_type 1319 +#define _tmp_69_type 1320 +#define _tmp_70_type 1321 +#define _loop0_72_type 1322 +#define _gather_71_type 1323 +#define _loop0_74_type 1324 +#define _gather_73_type 1325 +#define _tmp_75_type 1326 +#define _loop0_77_type 1327 +#define _gather_76_type 1328 +#define _loop0_79_type 1329 +#define _gather_78_type 1330 +#define _loop0_81_type 1331 +#define _gather_80_type 1332 +#define _loop1_82_type 1333 +#define _loop1_83_type 1334 +#define _loop0_85_type 1335 +#define _gather_84_type 1336 +#define _loop1_86_type 1337 +#define _loop1_87_type 1338 +#define _loop1_88_type 1339 +#define _tmp_89_type 1340 +#define _loop0_91_type 1341 +#define _gather_90_type 1342 +#define _tmp_92_type 1343 +#define _tmp_93_type 1344 +#define _tmp_94_type 1345 +#define _tmp_95_type 1346 +#define _tmp_96_type 1347 +#define _tmp_97_type 1348 +#define _loop0_98_type 1349 +#define _loop0_99_type 1350 +#define _loop0_100_type 1351 +#define _loop1_101_type 1352 +#define _loop0_102_type 1353 +#define _loop1_103_type 1354 +#define _loop1_104_type 1355 +#define _loop1_105_type 1356 +#define _loop0_106_type 1357 +#define _loop1_107_type 1358 +#define _loop0_108_type 1359 +#define _loop1_109_type 1360 +#define _loop0_110_type 1361 +#define _loop1_111_type 1362 +#define _loop0_112_type 1363 +#define _loop0_113_type 1364 +#define _loop1_114_type 1365 +#define _tmp_115_type 1366 +#define _loop0_117_type 1367 +#define _gather_116_type 1368 +#define _loop1_118_type 1369 +#define _loop0_119_type 1370 +#define _loop0_120_type 1371 +#define _tmp_121_type 1372 +#define _loop0_123_type 1373 +#define _gather_122_type 1374 +#define _tmp_124_type 1375 +#define _loop0_126_type 1376 +#define _gather_125_type 1377 +#define _loop0_128_type 1378 +#define _gather_127_type 1379 +#define _loop0_130_type 1380 +#define _gather_129_type 1381 +#define _loop0_132_type 1382 +#define _gather_131_type 1383 +#define _loop0_133_type 1384 +#define _loop0_135_type 1385 +#define _gather_134_type 1386 +#define _loop1_136_type 1387 +#define _tmp_137_type 1388 +#define _loop0_139_type 1389 +#define _gather_138_type 1390 +#define _loop0_141_type 1391 +#define _gather_140_type 1392 +#define _loop0_143_type 1393 +#define _gather_142_type 1394 +#define _loop0_145_type 1395 +#define _gather_144_type 1396 +#define _loop0_147_type 1397 +#define _gather_146_type 1398 +#define _tmp_148_type 1399 +#define _tmp_149_type 1400 +#define _loop0_151_type 1401 +#define _gather_150_type 1402 +#define _tmp_152_type 1403 +#define _tmp_153_type 1404 +#define _tmp_154_type 1405 +#define _tmp_155_type 1406 +#define _tmp_156_type 1407 +#define _tmp_157_type 1408 +#define _tmp_158_type 1409 +#define _tmp_159_type 1410 +#define _tmp_160_type 1411 +#define _tmp_161_type 1412 +#define _loop0_162_type 1413 +#define _loop0_163_type 1414 +#define _loop0_164_type 1415 +#define _tmp_165_type 1416 +#define _tmp_166_type 1417 +#define _tmp_167_type 1418 +#define _tmp_168_type 1419 +#define _loop0_169_type 1420 +#define _loop0_170_type 1421 +#define _loop0_171_type 1422 +#define _loop1_172_type 1423 +#define _tmp_173_type 1424 +#define _loop0_174_type 1425 +#define _tmp_175_type 1426 +#define _loop0_176_type 1427 +#define _loop1_177_type 1428 +#define _tmp_178_type 1429 +#define _tmp_179_type 1430 +#define _tmp_180_type 1431 +#define _loop0_181_type 1432 +#define _tmp_182_type 1433 +#define _tmp_183_type 1434 +#define _loop1_184_type 1435 +#define _tmp_185_type 1436 +#define _loop0_186_type 1437 +#define _loop0_187_type 1438 +#define _loop0_188_type 1439 +#define _loop0_190_type 1440 +#define _gather_189_type 1441 +#define _tmp_191_type 1442 +#define _loop0_192_type 1443 +#define _tmp_193_type 1444 +#define _loop0_194_type 1445 +#define _loop1_195_type 1446 +#define _loop1_196_type 1447 +#define _tmp_197_type 1448 +#define _tmp_198_type 1449 +#define _loop0_199_type 1450 +#define _tmp_200_type 1451 +#define _tmp_201_type 1452 +#define _tmp_202_type 1453 +#define _tmp_203_type 1454 +#define _loop0_205_type 1455 +#define _gather_204_type 1456 +#define _loop0_207_type 1457 +#define _gather_206_type 1458 +#define _loop0_209_type 1459 +#define _gather_208_type 1460 +#define _loop0_211_type 1461 +#define _gather_210_type 1462 +#define _loop0_213_type 1463 +#define _gather_212_type 1464 +#define _tmp_214_type 1465 +#define _loop0_215_type 1466 +#define _loop1_216_type 1467 +#define _tmp_217_type 1468 +#define _loop0_218_type 1469 +#define _loop1_219_type 1470 +#define _tmp_220_type 1471 +#define _tmp_221_type 1472 +#define _tmp_222_type 1473 +#define _tmp_223_type 1474 +#define _tmp_224_type 1475 +#define _tmp_225_type 1476 +#define _tmp_226_type 1477 +#define _tmp_227_type 1478 +#define _tmp_228_type 1479 +#define _tmp_229_type 1480 +#define _tmp_230_type 1481 +#define _loop0_232_type 1482 +#define _gather_231_type 1483 +#define _tmp_233_type 1484 +#define _tmp_234_type 1485 +#define _tmp_235_type 1486 +#define _tmp_236_type 1487 +#define _tmp_237_type 1488 +#define _tmp_238_type 1489 +#define _tmp_239_type 1490 +#define _loop0_240_type 1491 +#define _tmp_241_type 1492 +#define _tmp_242_type 1493 +#define _tmp_243_type 1494 +#define _tmp_244_type 1495 +#define _tmp_245_type 1496 +#define _tmp_246_type 1497 +#define _tmp_247_type 1498 +#define _tmp_248_type 1499 +#define _tmp_249_type 1500 +#define _tmp_250_type 1501 +#define _tmp_251_type 1502 +#define _tmp_252_type 1503 +#define _tmp_253_type 1504 +#define _tmp_254_type 1505 +#define _tmp_255_type 1506 +#define _tmp_256_type 1507 +#define _tmp_257_type 1508 +#define _tmp_258_type 1509 +#define _tmp_259_type 1510 +#define _tmp_260_type 1511 +#define _tmp_261_type 1512 +#define _tmp_262_type 1513 +#define _tmp_263_type 1514 +#define _tmp_264_type 1515 +#define _tmp_265_type 1516 +#define _loop0_266_type 1517 +#define _tmp_267_type 1518 +#define _tmp_268_type 1519 +#define _tmp_269_type 1520 +#define _tmp_270_type 1521 +#define _tmp_271_type 1522 +#define _tmp_272_type 1523 +#define _loop0_274_type 1524 +#define _gather_273_type 1525 +#define _tmp_275_type 1526 +#define _tmp_276_type 1527 +#define _tmp_277_type 1528 +#define _tmp_278_type 1529 +#define _tmp_279_type 1530 +#define _tmp_280_type 1531 +#define _tmp_281_type 1532 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -843,7 +842,6 @@ static void *invalid_for_target_rule(Parser *p); static void *invalid_group_rule(Parser *p); static void *invalid_import_rule(Parser *p); static void *invalid_import_from_targets_rule(Parser *p); -static void *invalid_compound_stmt_rule(Parser *p); static void *invalid_with_stmt_rule(Parser *p); static void *invalid_with_stmt_indent_rule(Parser *p); static void *invalid_try_stmt_rule(Parser *p); @@ -2062,7 +2060,6 @@ simple_stmt_rule(Parser *p) } // compound_stmt: -// | invalid_compound_stmt // | &('def' | '@' | 'async') function_def // | &'if' if_stmt // | &('class' | '@') class_def @@ -2083,25 +2080,6 @@ compound_stmt_rule(Parser *p) } stmt_ty _res = NULL; int _mark = p->mark; - if (p->call_invalid_rules) { // invalid_compound_stmt - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_compound_stmt")); - void *invalid_compound_stmt_var; - if ( - (invalid_compound_stmt_var = invalid_compound_stmt_rule(p)) // invalid_compound_stmt - ) - { - D(fprintf(stderr, "%*c+ compound_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_compound_stmt")); - _res = invalid_compound_stmt_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s compound_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_compound_stmt")); - } { // &('def' | '@' | 'async') function_def if (p->error_indicator) { p->level--; @@ -2131,7 +2109,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'if' if_stmt")); stmt_ty if_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 662) // token='if' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 660) // token='if' && (if_stmt_var = if_stmt_rule(p)) // if_stmt ) @@ -2215,7 +2193,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'try' try_stmt")); stmt_ty try_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 644) // token='try' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 642) // token='try' && (try_stmt_var = try_stmt_rule(p)) // try_stmt ) @@ -2236,7 +2214,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'while' while_stmt")); stmt_ty while_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 667) // token='while' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 665) // token='while' && (while_stmt_var = while_stmt_rule(p)) // while_stmt ) @@ -4376,7 +4354,7 @@ class_def_raw_rule(Parser *p) asdl_stmt_seq* c; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 679)) // token='class' + (_keyword = _PyPegen_expect_token(p, 677)) // token='class' && (a = _PyPegen_name_token(p)) // NAME && @@ -4543,7 +4521,7 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='def' + (_keyword = _PyPegen_expect_token(p, 675)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -4604,9 +4582,9 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 676)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 677)) // token='def' + (_keyword_1 = _PyPegen_expect_token(p, 675)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -5944,7 +5922,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 662)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -5989,7 +5967,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 662)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -6084,7 +6062,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 664)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 662)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6129,7 +6107,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 664)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 662)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6210,7 +6188,7 @@ else_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 665)) // token='else' + (_keyword = _PyPegen_expect_token(p, 663)) // token='else' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6289,7 +6267,7 @@ while_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 667)) // token='while' + (_keyword = _PyPegen_expect_token(p, 665)) // token='while' && (a = named_expression_rule(p)) // named_expression && @@ -6389,11 +6367,11 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 672)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 673)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='in' && (_cut_var = 1) && @@ -6451,13 +6429,13 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 676)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 672)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 670)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 673)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 671)) // token='in' && (_cut_var = 1) && @@ -6586,7 +6564,7 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 635)) // token='with' + (_keyword = _PyPegen_expect_token(p, 633)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -6637,7 +6615,7 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 635)) // token='with' + (_keyword = _PyPegen_expect_token(p, 633)) // token='with' && (a = (asdl_withitem_seq*)_gather_53_rule(p)) // ','.with_item+ && @@ -6686,9 +6664,9 @@ with_stmt_rule(Parser *p) asdl_withitem_seq* a; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 676)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 635)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 633)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -6738,9 +6716,9 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 676)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 635)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 633)) // token='with' && (a = (asdl_withitem_seq*)_gather_57_rule(p)) // ','.with_item+ && @@ -6826,7 +6804,7 @@ with_item_rule(Parser *p) if ( (e = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (t = star_target_rule(p)) // star_target && @@ -6951,7 +6929,7 @@ try_stmt_rule(Parser *p) asdl_stmt_seq* b; asdl_stmt_seq* f; if ( - (_keyword = _PyPegen_expect_token(p, 644)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6995,7 +6973,7 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 644)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7043,7 +7021,7 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 644)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7141,7 +7119,7 @@ except_block_rule(Parser *p) expr_ty e; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 657)) // token='except' + (_keyword = _PyPegen_expect_token(p, 655)) // token='except' && (e = expression_rule(p)) // expression && @@ -7184,7 +7162,7 @@ except_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 657)) // token='except' + (_keyword = _PyPegen_expect_token(p, 655)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -7295,7 +7273,7 @@ except_star_block_rule(Parser *p) expr_ty e; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 657)) // token='except' + (_keyword = _PyPegen_expect_token(p, 655)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -7397,7 +7375,7 @@ finally_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 653)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 651)) // token='finally' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7705,7 +7683,7 @@ guard_rule(Parser *p) Token * _keyword; expr_ty guard; if ( - (_keyword = _PyPegen_expect_token(p, 662)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (guard = named_expression_rule(p)) // named_expression ) @@ -7900,7 +7878,7 @@ as_pattern_rule(Parser *p) if ( (pattern = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (target = pattern_capture_target_rule(p)) // pattern_capture_target ) @@ -11195,11 +11173,11 @@ expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 662)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 665)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 663)) // token='else' && (c = expression_rule(p)) // expression ) @@ -12081,7 +12059,7 @@ inversion_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='not' + (_keyword = _PyPegen_expect_token(p, 679)) // token='not' && (a = inversion_rule(p)) // inversion ) @@ -12735,9 +12713,9 @@ notin_bitwise_or_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='not' + (_keyword = _PyPegen_expect_token(p, 679)) // token='not' && - (_keyword_1 = _PyPegen_expect_token(p, 673)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -12783,7 +12761,7 @@ in_bitwise_or_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 673)) // token='in' + (_keyword = _PyPegen_expect_token(p, 671)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -12832,7 +12810,7 @@ isnot_bitwise_or_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 589)) // token='is' && - (_keyword_1 = _PyPegen_expect_token(p, 681)) // token='not' + (_keyword_1 = _PyPegen_expect_token(p, 679)) // token='not' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -17003,13 +16981,13 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 676)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 672)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 670)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 673)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 671)) // token='in' && (_cut_var = 1) && @@ -17048,11 +17026,11 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 672)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 673)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='in' && (_cut_var = 1) && @@ -20353,11 +20331,11 @@ expression_without_invalid_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 662)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 665)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 663)) // token='else' && (c = expression_rule(p)) // expression ) @@ -20623,7 +20601,7 @@ invalid_expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 662)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (b = disjunction_rule(p)) // disjunction && @@ -22561,7 +22539,7 @@ invalid_with_item_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (a = expression_rule(p)) // expression && @@ -22611,13 +22589,13 @@ invalid_for_if_clause_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_tmp_203_var; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 672)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' && (_tmp_203_var = _tmp_203_rule(p)) // bitwise_or ((',' bitwise_or))* ','? && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 673) // token='in' + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 671) // token='in' ) { D(fprintf(stderr, "%*c+ invalid_for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); @@ -22663,9 +22641,9 @@ invalid_for_target_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings expr_ty a; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 672)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' && (a = star_expressions_rule(p)) // star_expressions ) @@ -22923,82 +22901,6 @@ invalid_import_from_targets_rule(Parser *p) return _res; } -// invalid_compound_stmt: 'elif' named_expression ':' | 'else' ':' -static void * -invalid_compound_stmt_rule(Parser *p) -{ - if (p->level++ == MAXSTACK) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // 'elif' named_expression ':' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> invalid_compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'elif' named_expression ':'")); - Token * _literal; - Token * a; - expr_ty named_expression_var; - if ( - (a = _PyPegen_expect_token(p, 664)) // token='elif' - && - (named_expression_var = named_expression_rule(p)) // named_expression - && - (_literal = _PyPegen_expect_token(p, 11)) // token=':' - ) - { - D(fprintf(stderr, "%*c+ invalid_compound_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'elif' named_expression ':'")); - _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "'elif' must match an if-statement here" ); - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s invalid_compound_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'elif' named_expression ':'")); - } - { // 'else' ':' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> invalid_compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else' ':'")); - Token * _literal; - Token * a; - if ( - (a = _PyPegen_expect_token(p, 665)) // token='else' - && - (_literal = _PyPegen_expect_token(p, 11)) // token=':' - ) - { - D(fprintf(stderr, "%*c+ invalid_compound_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else' ':'")); - _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "'else' must match a valid statement here" ); - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s invalid_compound_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'else' ':'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - // invalid_with_stmt: // | 'async'? 'with' ','.(expression ['as' star_target])+ NEWLINE // | 'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE @@ -23026,9 +22928,9 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 635)) // token='with' + (_keyword = _PyPegen_expect_token(p, 633)) // token='with' && (_gather_206_var = _gather_206_rule(p)) // ','.(expression ['as' star_target])+ && @@ -23064,9 +22966,9 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var_1); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 635)) // token='with' + (_keyword = _PyPegen_expect_token(p, 633)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -23126,9 +23028,9 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 635)) // token='with' + (a = _PyPegen_expect_token(p, 633)) // token='with' && (_gather_210_var = _gather_210_rule(p)) // ','.(expression ['as' star_target])+ && @@ -23169,9 +23071,9 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 635)) // token='with' + (a = _PyPegen_expect_token(p, 633)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -23234,7 +23136,7 @@ invalid_try_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 644)) // token='try' + (a = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23266,7 +23168,7 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; asdl_stmt_seq* block_var; if ( - (_keyword = _PyPegen_expect_token(p, 644)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23305,7 +23207,7 @@ invalid_try_stmt_rule(Parser *p) Token * b; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 644)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23313,7 +23215,7 @@ invalid_try_stmt_rule(Parser *p) && (_loop1_216_var = _loop1_216_rule(p)) // except_block+ && - (a = _PyPegen_expect_token(p, 657)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (b = _PyPegen_expect_token(p, 16)) // token='*' && @@ -23352,7 +23254,7 @@ invalid_try_stmt_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings Token * a; if ( - (_keyword = _PyPegen_expect_token(p, 644)) // token='try' + (_keyword = _PyPegen_expect_token(p, 642)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23360,7 +23262,7 @@ invalid_try_stmt_rule(Parser *p) && (_loop1_219_var = _loop1_219_rule(p)) // except_star_block+ && - (a = _PyPegen_expect_token(p, 657)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (_opt_var = _tmp_220_rule(p), !p->error_indicator) // [expression ['as' NAME]] && @@ -23419,7 +23321,7 @@ invalid_except_stmt_rule(Parser *p) expr_ty a; expr_ty expressions_var; if ( - (_keyword = _PyPegen_expect_token(p, 657)) // token='except' + (_keyword = _PyPegen_expect_token(p, 655)) // token='except' && (_opt_var = _PyPegen_expect_token(p, 16), !p->error_indicator) // '*'? && @@ -23461,7 +23363,7 @@ invalid_except_stmt_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 657)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (_opt_var = _PyPegen_expect_token(p, 16), !p->error_indicator) // '*'? && @@ -23494,7 +23396,7 @@ invalid_except_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 657)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23522,7 +23424,7 @@ invalid_except_stmt_rule(Parser *p) void *_tmp_223_var; Token * a; if ( - (a = _PyPegen_expect_token(p, 657)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -23571,7 +23473,7 @@ invalid_finally_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 653)) // token='finally' + (a = _PyPegen_expect_token(p, 651)) // token='finally' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23627,7 +23529,7 @@ invalid_except_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 657)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (expression_var = expression_rule(p)) // expression && @@ -23663,7 +23565,7 @@ invalid_except_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 657)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23719,7 +23621,7 @@ invalid_except_star_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 657)) // token='except' + (a = _PyPegen_expect_token(p, 655)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -23958,7 +23860,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (a = _PyPegen_expect_soft_keyword(p, "_")) // soft_keyword='"_"' ) @@ -23988,7 +23890,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && _PyPegen_lookahead_with_name(0, _PyPegen_name_token, p) && @@ -24142,7 +24044,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 662)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -24173,7 +24075,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty a_1; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 662)) // token='if' + (a = _PyPegen_expect_token(p, 660)) // token='if' && (a_1 = named_expression_rule(p)) // named_expression && @@ -24228,7 +24130,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 664)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 662)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -24259,7 +24161,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 664)) // token='elif' + (a = _PyPegen_expect_token(p, 662)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -24312,7 +24214,7 @@ invalid_else_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 665)) // token='else' + (a = _PyPegen_expect_token(p, 663)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24365,7 +24267,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 667)) // token='while' + (_keyword = _PyPegen_expect_token(p, 665)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -24396,7 +24298,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 667)) // token='while' + (a = _PyPegen_expect_token(p, 665)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -24455,13 +24357,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 672)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 673)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 671)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -24496,13 +24398,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 672)) // token='for' + (a = _PyPegen_expect_token(p, 670)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword = _PyPegen_expect_token(p, 673)) // token='in' + (_keyword = _PyPegen_expect_token(p, 671)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -24568,9 +24470,9 @@ invalid_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 677)) // token='def' + (a = _PyPegen_expect_token(p, 675)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24627,9 +24529,9 @@ invalid_def_raw_rule(Parser *p) asdl_stmt_seq* block_var; expr_ty name_var; if ( - (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 677)) // token='def' + (_keyword = _PyPegen_expect_token(p, 675)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24693,7 +24595,7 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 679)) // token='class' + (_keyword = _PyPegen_expect_token(p, 677)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24732,7 +24634,7 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 679)) // token='class' + (a = _PyPegen_expect_token(p, 677)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -25550,7 +25452,7 @@ invalid_arithmetic_rule(Parser *p) && (_tmp_243_var = _tmp_243_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' && - (a = _PyPegen_expect_token(p, 681)) // token='not' + (a = _PyPegen_expect_token(p, 679)) // token='not' && (b = inversion_rule(p)) // inversion ) @@ -25599,7 +25501,7 @@ invalid_factor_rule(Parser *p) if ( (_tmp_244_var = _tmp_244_rule(p)) // '+' | '-' | '~' && - (a = _PyPegen_expect_token(p, 681)) // token='not' + (a = _PyPegen_expect_token(p, 679)) // token='not' && (b = factor_rule(p)) // factor ) @@ -26070,7 +25972,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='def' + (_keyword = _PyPegen_expect_token(p, 675)) // token='def' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); @@ -26108,7 +26010,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 676)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -26146,7 +26048,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 679)) // token='class' + (_keyword = _PyPegen_expect_token(p, 677)) // token='class' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); @@ -26203,7 +26105,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'with'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 635)) // token='with' + (_keyword = _PyPegen_expect_token(p, 633)) // token='with' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'with'")); @@ -26222,7 +26124,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 676)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -26260,7 +26162,7 @@ _tmp_10_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'for'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 672)) // token='for' + (_keyword = _PyPegen_expect_token(p, 670)) // token='for' ) { D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for'")); @@ -26279,7 +26181,7 @@ _tmp_10_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 676)) // token='async' + (_keyword = _PyPegen_expect_token(p, 674)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -27303,7 +27205,7 @@ _tmp_28_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -27466,7 +27368,7 @@ _tmp_31_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -29453,7 +29355,7 @@ _tmp_62_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -29499,7 +29401,7 @@ _tmp_63_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) @@ -35341,7 +35243,7 @@ _tmp_158_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 665)) // token='else' + (_keyword = _PyPegen_expect_token(p, 663)) // token='else' ) { D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); @@ -38922,7 +38824,7 @@ _tmp_214_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 657)) // token='except' + (_keyword = _PyPegen_expect_token(p, 655)) // token='except' ) { D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); @@ -38941,7 +38843,7 @@ _tmp_214_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_214[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 653)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 651)) // token='finally' ) { D(fprintf(stderr, "%*c+ _tmp_214[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); @@ -39119,7 +39021,7 @@ _tmp_217_rule(Parser *p) Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) @@ -39341,7 +39243,7 @@ _tmp_221_rule(Parser *p) Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) @@ -39382,7 +39284,7 @@ _tmp_222_rule(Parser *p) Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) @@ -39480,7 +39382,7 @@ _tmp_224_rule(Parser *p) Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) @@ -39521,7 +39423,7 @@ _tmp_225_rule(Parser *p) Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) @@ -41216,7 +41118,7 @@ _tmp_255_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 662)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (z = disjunction_rule(p)) // disjunction ) @@ -41262,7 +41164,7 @@ _tmp_256_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 662)) // token='if' + (_keyword = _PyPegen_expect_token(p, 660)) // token='if' && (z = disjunction_rule(p)) // disjunction ) @@ -41975,7 +41877,7 @@ _tmp_271_rule(Parser *p) Token * _keyword; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME ) @@ -42234,7 +42136,7 @@ _tmp_276_rule(Parser *p) Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) @@ -42275,7 +42177,7 @@ _tmp_277_rule(Parser *p) Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) @@ -42316,7 +42218,7 @@ _tmp_278_rule(Parser *p) Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) @@ -42357,7 +42259,7 @@ _tmp_279_rule(Parser *p) Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='as' + (_keyword = _PyPegen_expect_token(p, 658)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) From 8d63c8d47b9edd8ac2f0b395b2fa0ae5f571252d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 4 Jun 2024 02:36:28 -0400 Subject: [PATCH 358/903] gh-106531: Apply changes from importlib_resources 6.3.2 (#117054) Apply changes from importlib_resources 6.3.2. --- Lib/importlib/resources/_common.py | 2 + Lib/importlib/resources/readers.py | 54 ++++++- .../resources/data01/subdirectory/binary.file | Bin 4 -> 4 bytes .../namespacedata01/subdirectory/binary.file | 1 + .../test_importlib/resources/test_contents.py | 2 +- .../test_importlib/resources/test_custom.py | 6 +- .../test_importlib/resources/test_files.py | 14 +- .../test_importlib/resources/test_open.py | 6 +- .../test_importlib/resources/test_path.py | 12 +- .../test_importlib/resources/test_read.py | 29 +++- .../test_importlib/resources/test_reader.py | 29 ++-- .../test_importlib/resources/test_resource.py | 134 ++++++++---------- Lib/test/test_importlib/resources/util.py | 51 +++---- Lib/test/test_importlib/resources/zip.py | 30 ++++ Makefile.pre.in | 1 + ...-03-19-21-41-31.gh-issue-106531.Mgd--6.rst | 6 + 16 files changed, 231 insertions(+), 146 deletions(-) create mode 100644 Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file create mode 100755 Lib/test/test_importlib/resources/zip.py create mode 100644 Misc/NEWS.d/next/Library/2024-03-19-21-41-31.gh-issue-106531.Mgd--6.rst diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index e18082fb3d26a0..ca5b06743b46a6 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -25,6 +25,8 @@ def package_to_anchor(func): >>> files('a', 'b') Traceback (most recent call last): TypeError: files() takes from 0 to 1 positional arguments but 2 were given + + Remove this compatibility in Python 3.14. """ undefined = object() diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index c3cdf769cbecb0..b86cdeff57c4c2 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -1,7 +1,10 @@ import collections +import contextlib import itertools import pathlib import operator +import re +import warnings import zipfile from . import abc @@ -62,7 +65,7 @@ class MultiplexedPath(abc.Traversable): """ def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + self._paths = list(map(_ensure_traversable, remove_duplicates(paths))) if not self._paths: message = 'MultiplexedPath must contain at least one path' raise FileNotFoundError(message) @@ -130,7 +133,36 @@ class NamespaceReader(abc.TraversableResources): def __init__(self, namespace_path): if 'NamespacePath' not in str(namespace_path): raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) + self.path = MultiplexedPath(*map(self._resolve, namespace_path)) + + @classmethod + def _resolve(cls, path_str) -> abc.Traversable: + r""" + Given an item from a namespace path, resolve it to a Traversable. + + path_str might be a directory on the filesystem or a path to a + zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or + ``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``. + """ + (dir,) = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) + return dir + + @classmethod + def _candidate_paths(cls, path_str): + yield pathlib.Path(path_str) + yield from cls._resolve_zip_path(path_str) + + @staticmethod + def _resolve_zip_path(path_str): + for match in reversed(list(re.finditer(r'[\\/]', path_str))): + with contextlib.suppress( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + PermissionError, + ): + inner = path_str[match.end() :].replace('\\', '/') + '/' + yield zipfile.Path(path_str[: match.start()], inner.lstrip('/')) def resource_path(self, resource): """ @@ -142,3 +174,21 @@ def resource_path(self, resource): def files(self): return self.path + + +def _ensure_traversable(path): + """ + Convert deprecated string arguments to traversables (pathlib.Path). + + Remove with Python 3.15. + """ + if not isinstance(path, str): + return path + + warnings.warn( + "String arguments are deprecated. Pass a Traversable instead.", + DeprecationWarning, + stacklevel=3, + ) + + return pathlib.Path(path) diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file b/Lib/test/test_importlib/resources/data01/subdirectory/binary.file index eaf36c1daccfdf325514461cd1a2ffbc139b5464..5bd8bb897b13225c93a1d26baa88c96b7bd5d817 100644 GIT binary patch literal 4 LcmZQ!Wn%{b05$*@ literal 4 LcmZQzWMT#Y01f~L diff --git a/Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file b/Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file new file mode 100644 index 00000000000000..100f50643d8d21 --- /dev/null +++ b/Lib/test/test_importlib/resources/namespacedata01/subdirectory/binary.file @@ -0,0 +1 @@ +  \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/test_contents.py b/Lib/test/test_importlib/resources/test_contents.py index 1a13f043a86f03..beab67ccc21680 100644 --- a/Lib/test/test_importlib/resources/test_contents.py +++ b/Lib/test/test_importlib/resources/test_contents.py @@ -31,8 +31,8 @@ class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): class ContentsNamespaceTests(ContentsTests, unittest.TestCase): expected = { # no __init__ because of namespace design - # no subdirectory as incidental difference in fixture 'binary.file', + 'subdirectory', 'utf-16.file', 'utf-8.file', } diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py index 73127209a2761b..640f90fc0dd91a 100644 --- a/Lib/test/test_importlib/resources/test_custom.py +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -5,6 +5,7 @@ from test.support import os_helper from importlib import resources +from importlib.resources import abc from importlib.resources.abc import TraversableResources, ResourceReader from . import util @@ -39,8 +40,9 @@ def setUp(self): self.addCleanup(self.fixtures.close) def test_custom_loader(self): - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + temp_dir = pathlib.Path(self.fixtures.enter_context(os_helper.temp_dir())) loader = SimpleLoader(MagicResources(temp_dir)) pkg = util.create_package_from_loader(loader) files = resources.files(pkg) - assert files is temp_dir + assert isinstance(files, abc.Traversable) + assert list(files.iterdir()) == [] diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 26c8b04e44c3b9..7df6d03ead7480 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,4 +1,3 @@ -import typing import textwrap import unittest import warnings @@ -32,13 +31,14 @@ def test_read_text(self): actual = files.joinpath('utf-8.file').read_text(encoding='utf-8') assert actual == 'Hello, UTF-8 world!\n' - @unittest.skipUnless( - hasattr(typing, 'runtime_checkable'), - "Only suitable when typing supports runtime_checkable", - ) def test_traversable(self): assert isinstance(resources.files(self.data), Traversable) + def test_joinpath_with_multiple_args(self): + files = resources.files(self.data) + binfile = files.joinpath('subdirectory', 'binary.file') + self.assertTrue(binfile.is_file()) + def test_old_parameter(self): """ Files used to take a 'package' parameter. Make sure anyone @@ -64,6 +64,10 @@ def setUp(self): self.data = namespacedata01 +class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' + + class SiteDir: def setUp(self): self.fixtures = contextlib.ExitStack() diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 86becb4bfaad37..3b6b2142ef47b1 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -24,7 +24,7 @@ def test_open_binary(self): target = resources.files(self.data) / 'binary.file' with target.open('rb') as fp: result = fp.read() - self.assertEqual(result, b'\x00\x01\x02\x03') + self.assertEqual(result, bytes(range(4))) def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' @@ -81,5 +81,9 @@ class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): pass +class OpenNamespaceZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index 34a6bdd2d58b91..90b22905ab8692 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -1,4 +1,5 @@ import io +import pathlib import unittest from importlib import resources @@ -15,18 +16,13 @@ def execute(self, package, path): class PathTests: def test_reading(self): """ - Path should be readable. - - Test also implicitly verifies the returned object is a pathlib.Path - instance. + Path should be readable and a pathlib.Path instance. """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: + self.assertIsInstance(path, pathlib.Path) self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) - # pathlib.Path.read_text() was introduced in Python 3.5. - with path.open('r', encoding='utf-8') as file: - text = file.read() - self.assertEqual('Hello, UTF-8 world!\n', text) + self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) class PathDiskTests(PathTests, unittest.TestCase): diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 088982681e8b0c..984feecbb9ed69 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -18,7 +18,7 @@ def execute(self, package, path): class ReadTests: def test_read_bytes(self): result = resources.files(self.data).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4))) def test_read_text_default_encoding(self): result = ( @@ -57,17 +57,15 @@ class ReadDiskTests(ReadTests, unittest.TestCase): class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) def test_read_submodule_resource_by_name(self): result = ( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .read_bytes() + resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() ) - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) class ReadNamespaceTests(ReadTests, unittest.TestCase): @@ -77,5 +75,22 @@ def setUp(self): self.data = namespacedata01 +class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' + + def test_read_submodule_resource(self): + submodule = import_module('namespacedata01.subdirectory') + result = resources.files(submodule).joinpath('binary.file').read_bytes() + self.assertEqual(result, bytes(range(12, 16))) + + def test_read_submodule_resource_by_name(self): + result = ( + resources.files('namespacedata01.subdirectory') + .joinpath('binary.file') + .read_bytes() + ) + self.assertEqual(result, bytes(range(12, 16))) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index 8670f72a334585..dac9c2a892ffd2 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -10,8 +10,7 @@ class MultiplexedPathTest(unittest.TestCase): @classmethod def setUpClass(cls): - path = pathlib.Path(__file__).parent / 'namespacedata01' - cls.folder = str(path) + cls.folder = pathlib.Path(__file__).parent / 'namespacedata01' def test_init_no_paths(self): with self.assertRaises(FileNotFoundError): @@ -19,7 +18,7 @@ def test_init_no_paths(self): def test_init_file(self): with self.assertRaises(NotADirectoryError): - MultiplexedPath(os.path.join(self.folder, 'binary.file')) + MultiplexedPath(self.folder / 'binary.file') def test_iterdir(self): contents = {path.name for path in MultiplexedPath(self.folder).iterdir()} @@ -27,10 +26,12 @@ def test_iterdir(self): contents.remove('__pycache__') except (KeyError, ValueError): pass - self.assertEqual(contents, {'binary.file', 'utf-16.file', 'utf-8.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} + ) def test_iterdir_duplicate(self): - data01 = os.path.abspath(os.path.join(__file__, '..', 'data01')) + data01 = pathlib.Path(__file__).parent.joinpath('data01') contents = { path.name for path in MultiplexedPath(self.folder, data01).iterdir() } @@ -60,17 +61,17 @@ def test_open_file(self): path.open() def test_join_path(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') + data01 = pathlib.Path(__file__).parent.joinpath('data01') + prefix = str(data01.parent) path = MultiplexedPath(self.folder, data01) self.assertEqual( str(path.joinpath('binary.file'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'binary.file'), ) - self.assertEqual( - str(path.joinpath('subdirectory'))[len(prefix) + 1 :], - os.path.join('data01', 'subdirectory'), - ) + sub = path.joinpath('subdirectory') + assert isinstance(sub, MultiplexedPath) + assert 'namespacedata01' in str(sub) + assert 'data01' in str(sub) self.assertEqual( str(path.joinpath('imaginary'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'imaginary'), @@ -82,9 +83,9 @@ def test_join_path_compound(self): assert not path.joinpath('imaginary/foo.py').exists() def test_join_path_common_subdir(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - data02 = os.path.join(prefix, 'data02') + data01 = pathlib.Path(__file__).parent.joinpath('data01') + data02 = pathlib.Path(__file__).parent.joinpath('data02') + prefix = str(data01.parent) path = MultiplexedPath(data01, data02) self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) self.assertEqual( diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index 6f75cf57f03d02..d1d45d9b4617f3 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,15 +1,10 @@ -import contextlib import sys import unittest -import uuid import pathlib from . import data01 -from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module -from test.support import import_helper, os_helper -from test.support.os_helper import unlink class ResourceTests: @@ -89,34 +84,32 @@ def test_package_has_no_reader_fallback(self): class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore + ZIP_MODULE = 'data01' def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) def test_read_submodule_resource_by_name(self): self.assertTrue( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .is_file() + resources.files('data01.subdirectory').joinpath('binary.file').is_file() ) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertEqual( names(resources.files(submodule)), {'__init__.py', 'binary.file'} ) def test_submodule_contents_by_name(self): self.assertEqual( - names(resources.files('ziptestdata.subdirectory')), + names(resources.files('data01.subdirectory')), {'__init__.py', 'binary.file'}, ) def test_as_file_directory(self): - with resources.as_file(resources.files('ziptestdata')) as data: - assert data.name == 'ziptestdata' + with resources.as_file(resources.files('data01')) as data: + assert data.name == 'data01' assert data.is_dir() assert data.joinpath('subdirectory').is_dir() assert len(list(data.iterdir())) @@ -124,7 +117,7 @@ def test_as_file_directory(self): class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore + ZIP_MODULE = 'data02' def test_unrelated_contents(self): """ @@ -132,93 +125,48 @@ def test_unrelated_contents(self): distinct resources. Ref python/importlib_resources#44. """ self.assertEqual( - names(resources.files('ziptestdata.one')), + names(resources.files('data02.one')), {'__init__.py', 'resource1.txt'}, ) self.assertEqual( - names(resources.files('ziptestdata.two')), + names(resources.files('data02.two')), {'__init__.py', 'resource2.txt'}, ) -@contextlib.contextmanager -def zip_on_path(dir): - data_path = pathlib.Path(zipdata01.__file__) - source_zip_path = data_path.parent.joinpath('ziptestdata.zip') - zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' - zip_path.write_bytes(source_zip_path.read_bytes()) - sys.path.append(str(zip_path)) - import_module('ziptestdata') - - try: - yield - finally: - with contextlib.suppress(ValueError): - sys.path.remove(str(zip_path)) - - with contextlib.suppress(KeyError): - del sys.path_importer_cache[str(zip_path)] - del sys.modules['ziptestdata'] - - with contextlib.suppress(OSError): - unlink(zip_path) - - -class DeletingZipsTest(unittest.TestCase): +class DeletingZipsTest(util.ZipSetupBase, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(zip_on_path(temp_dir)) - def test_iterdir_does_not_keep_open(self): - [item.name for item in resources.files('ziptestdata').iterdir()] + [item.name for item in resources.files('data01').iterdir()] def test_is_file_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').is_file() + resources.files('data01').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('not-present').is_file() + resources.files('data01').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - resources.as_file(resources.files('ziptestdata') / 'binary.file') + resources.as_file(resources.files('data01') / 'binary.file') def test_entered_path_does_not_keep_open(self): """ Mimic what certifi does on import to make its bundle available for the process duration. """ - resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() + resources.as_file(resources.files('data01') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').read_bytes() + resources.files('data01').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('utf-8.file').read_text( - encoding='utf-8' - ) + resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') -class ResourceFromNamespaceTest01(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) - +class ResourceFromNamespaceTests: def test_is_submodule_resource(self): self.assertTrue( resources.files(import_module('namespacedata01')) @@ -237,7 +185,9 @@ def test_submodule_contents(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) def test_submodule_contents_by_name(self): contents = names(resources.files('namespacedata01')) @@ -245,7 +195,45 @@ def test_submodule_contents_by_name(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) + + def test_submodule_sub_contents(self): + contents = names(resources.files(import_module('namespacedata01.subdirectory'))) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + def test_submodule_sub_contents_by_name(self): + contents = names(resources.files('namespacedata01.subdirectory')) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + +class ResourceFromNamespaceDiskTests(ResourceFromNamespaceTests, unittest.TestCase): + site_dir = str(pathlib.Path(__file__).parent) + + @classmethod + def setUpClass(cls): + sys.path.append(cls.site_dir) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.site_dir) + + +class ResourceFromNamespaceZipTests( + util.ZipSetupBase, + ResourceFromNamespaceTests, + unittest.TestCase, +): + ZIP_MODULE = 'namespacedata01' if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index dbe6ee81476699..d4bf3e6cc5dfdc 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -4,11 +4,12 @@ import sys import types import pathlib +import contextlib from . import data01 -from . import zipdata01 from importlib.resources.abc import ResourceReader -from test.support import import_helper +from test.support import import_helper, os_helper +from . import zip as zip_ from importlib.machinery import ModuleSpec @@ -141,39 +142,23 @@ def test_useless_loader(self): class ZipSetupBase: - ZIP_MODULE = None - - @classmethod - def setUpClass(cls): - data_path = pathlib.Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') - - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass - - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - - try: - del cls.data - del cls._zip_path - except AttributeError: - pass + ZIP_MODULE = 'data01' def setUp(self): - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + self.fixtures.enter_context(import_helper.isolated_modules()) + + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + modules = pathlib.Path(temp_dir) / 'zipped modules.zip' + src_path = pathlib.Path(__file__).parent.joinpath(self.ZIP_MODULE) + self.fixtures.enter_context( + import_helper.DirsOnSysPath(str(zip_.make_zip_file(src_path, modules))) + ) + + self.data = importlib.import_module(self.ZIP_MODULE) class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore + pass diff --git a/Lib/test/test_importlib/resources/zip.py b/Lib/test/test_importlib/resources/zip.py new file mode 100755 index 00000000000000..4dcf6facc770cb --- /dev/null +++ b/Lib/test/test_importlib/resources/zip.py @@ -0,0 +1,30 @@ +""" +Generate zip test data files. +""" + +import contextlib +import os +import pathlib +import zipfile + + +def make_zip_file(src, dst): + """ + Zip the files in src into a new zipfile at dst. + """ + with zipfile.ZipFile(dst, 'w') as zf: + for src_path, rel in walk(src): + dst_name = src.name / pathlib.PurePosixPath(rel.as_posix()) + zf.write(src_path, dst_name) + zipfile._path.CompleteDirs.inject(zf) + return dst + + +def walk(datapath): + for dirpath, dirnames, filenames in os.walk(datapath): + with contextlib.suppress(ValueError): + dirnames.remove('__pycache__') + for filename in filenames: + res = pathlib.Path(dirpath) / filename + rel = res.relative_to(datapath) + yield res, rel diff --git a/Makefile.pre.in b/Makefile.pre.in index 9a2fc34f030662..22dba279faa935 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2438,6 +2438,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/resources/data03/namespace/portion1 \ test/test_importlib/resources/data03/namespace/portion2 \ test/test_importlib/resources/namespacedata01 \ + test/test_importlib/resources/namespacedata01/subdirectory \ test/test_importlib/resources/zipdata01 \ test/test_importlib/resources/zipdata02 \ test/test_importlib/source \ diff --git a/Misc/NEWS.d/next/Library/2024-03-19-21-41-31.gh-issue-106531.Mgd--6.rst b/Misc/NEWS.d/next/Library/2024-03-19-21-41-31.gh-issue-106531.Mgd--6.rst new file mode 100644 index 00000000000000..6a5783c5ad9846 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-19-21-41-31.gh-issue-106531.Mgd--6.rst @@ -0,0 +1,6 @@ +In :mod:`importlib.resources`, sync with `importlib_resources 6.3.2 +`_, +including: ``MultiplexedPath`` now expects ``Traversable`` paths, +deprecating string arguments to ``MultiplexedPath``; Enabled support for +resources in namespace packages in zip files; Fixed ``NotADirectoryError`` +when calling files on a subdirectory of a namespace package. From a8f1152b70d707340b394689cd09aa0831da3601 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 4 Jun 2024 10:44:49 +0300 Subject: [PATCH 359/903] gh-119879: str.find(): Utilize last character gap for two-way periodic needles (#119880) --- ...-06-02-06-12-35.gh-issue-119879.Jit951.rst | 1 + Objects/stringlib/fastsearch.h | 63 ++++++++++--------- 2 files changed, 36 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-02-06-12-35.gh-issue-119879.Jit951.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-02-06-12-35.gh-issue-119879.Jit951.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-02-06-12-35.gh-issue-119879.Jit951.rst new file mode 100644 index 00000000000000..89de6b0299a35a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-02-06-12-35.gh-issue-119879.Jit951.rst @@ -0,0 +1 @@ +String search is now slightly faster for certain cases. It now utilizes last character gap (good suffix rule) for two-way periodic needles. diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index 257b7bd6788ad2..309ed1554f4699 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -256,7 +256,7 @@ STRINGLIB(_factorize)(const STRINGLIB_CHAR *needle, The local period of the cut is the minimal length of a string w such that (left endswith w or w endswith left) - and (right startswith w or w startswith left). + and (right startswith w or w startswith right). The Critical Factorization Theorem says that this maximal local period is the global period of the string. @@ -337,21 +337,20 @@ STRINGLIB(_preprocess)(const STRINGLIB_CHAR *needle, Py_ssize_t len_needle, if (p->is_periodic) { assert(p->cut <= len_needle/2); assert(p->cut < p->period); - p->gap = 0; // unused } else { // A lower bound on the period p->period = Py_MAX(p->cut, len_needle - p->cut) + 1; - // The gap between the last character and the previous - // occurrence of an equivalent character (modulo TABLE_SIZE) - p->gap = len_needle; - STRINGLIB_CHAR last = needle[len_needle - 1] & TABLE_MASK; - for (Py_ssize_t i = len_needle - 2; i >= 0; i--) { - STRINGLIB_CHAR x = needle[i] & TABLE_MASK; - if (x == last) { - p->gap = len_needle - 1 - i; - break; - } + } + // The gap between the last character and the previous + // occurrence of an equivalent character (modulo TABLE_SIZE) + p->gap = len_needle; + STRINGLIB_CHAR last = needle[len_needle - 1] & TABLE_MASK; + for (Py_ssize_t i = len_needle - 2; i >= 0; i--) { + STRINGLIB_CHAR x = needle[i] & TABLE_MASK; + if (x == last) { + p->gap = len_needle - 1 - i; + break; } } // Fill up a compressed Boyer-Moore "Bad Character" table @@ -383,6 +382,8 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, const STRINGLIB_CHAR *window; LOG("===== Two-way: \"%s\" in \"%s\". =====\n", needle, haystack); + Py_ssize_t gap = p->gap; + Py_ssize_t gap_jump_end = Py_MIN(len_needle, cut + gap); if (p->is_periodic) { LOG("Needle is periodic.\n"); Py_ssize_t memory = 0; @@ -408,8 +409,16 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, Py_ssize_t i = Py_MAX(cut, memory); for (; i < len_needle; i++) { if (needle[i] != window[i]) { - LOG("Right half does not match.\n"); - window_last += i - cut + 1; + if (i < gap_jump_end) { + LOG("Early right half mismatch: jump by gap.\n"); + assert(gap >= i - cut + 1); + window_last += gap; + } + else { + LOG("Late right half mismatch: jump by n (>gap)\n"); + assert(i - cut + 1 > gap); + window_last += i - cut + 1; + } memory = 0; goto periodicwindowloop; } @@ -442,10 +451,8 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, } } else { - Py_ssize_t gap = p->gap; period = Py_MAX(gap, period); LOG("Needle is not periodic.\n"); - Py_ssize_t gap_jump_end = Py_MIN(len_needle, cut + gap); windowloop: while (window_last < haystack_end) { for (;;) { @@ -463,19 +470,19 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, window = window_last - len_needle + 1; assert((window[len_needle - 1] & TABLE_MASK) == (needle[len_needle - 1] & TABLE_MASK)); - for (Py_ssize_t i = cut; i < gap_jump_end; i++) { - if (needle[i] != window[i]) { - LOG("Early right half mismatch: jump by gap.\n"); - assert(gap >= i - cut + 1); - window_last += gap; - goto windowloop; - } - } - for (Py_ssize_t i = gap_jump_end; i < len_needle; i++) { + Py_ssize_t i = cut; + for (; i < len_needle; i++) { if (needle[i] != window[i]) { - LOG("Late right half mismatch.\n"); - assert(i - cut + 1 > gap); - window_last += i - cut + 1; + if (i < gap_jump_end) { + LOG("Early right half mismatch: jump by gap.\n"); + assert(gap >= i - cut + 1); + window_last += gap; + } + else { + LOG("Late right half mismatch: jump by n (>gap)\n"); + assert(i - cut + 1 > gap); + window_last += i - cut + 1; + } goto windowloop; } } From 5c48eb0cc6c3e84aafda0a734a05ecec14fc0ccf Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 4 Jun 2024 09:17:45 +0100 Subject: [PATCH 360/903] gh-119070: Update test_shebang_executable_extension to always use non-installed version (GH-119846) --- Lib/test/test_launcher.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index 6d358ac6f16a27..58baae25df3df7 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -766,9 +766,9 @@ def test_shebang_command_in_venv(self): self.assertEqual(data["stdout"].strip(), f"{quote(exe)} arg1 {quote(script)}") def test_shebang_executable_extension(self): - with self.script('#! /usr/bin/env python3.12') as script: - data = self.run_py([script]) - expect = "# Search PATH for python3.12.exe" + with self.script('#! /usr/bin/env python3.99') as script: + data = self.run_py([script], expect_returncode=103) + expect = "# Search PATH for python3.99.exe" actual = [line.strip() for line in data["stderr"].splitlines() if line.startswith("# Search PATH")] self.assertEqual([expect], actual) From 26e5c6e8351adb1a77a88920ff33fc8ebee9a99e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 4 Jun 2024 11:23:55 +0200 Subject: [PATCH 361/903] gh-119613: Soft deprecate the Py_MEMCPY() macro (#120020) Use directly memcpy() instead. --- Include/pyport.h | 1 + .../next/C API/2024-06-04-10-58-20.gh-issue-119613.qOr9GF.rst | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2024-06-04-10-58-20.gh-issue-119613.qOr9GF.rst diff --git a/Include/pyport.h b/Include/pyport.h index 2ba81a4be42822..1f7a9b41e0ae2b 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -180,6 +180,7 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_LOCAL_INLINE(type) static inline type #endif +// Soft deprecated since Python 3.14, use memcpy() instead. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_MEMCPY memcpy #endif diff --git a/Misc/NEWS.d/next/C API/2024-06-04-10-58-20.gh-issue-119613.qOr9GF.rst b/Misc/NEWS.d/next/C API/2024-06-04-10-58-20.gh-issue-119613.qOr9GF.rst new file mode 100644 index 00000000000000..11f075b79e6f67 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-06-04-10-58-20.gh-issue-119613.qOr9GF.rst @@ -0,0 +1,2 @@ +Soft deprecate the :c:macro:`!Py_MEMCPY` macro: use directly ``memcpy()`` +instead. Patch by Victor Stinner. From 5a1205b641df133932ed4c65b9a4ff5724e89963 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 4 Jun 2024 11:39:07 +0200 Subject: [PATCH 362/903] gh-111499: Fix PYTHONMALLOCSTATS at Python exit (#120021) Call _PyObject_DebugMallocStats() earlier in Py_FinalizeEx(), before the interpreter is deleted. --- Python/pylifecycle.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 67bbbd01ca0c48..cbdf5c1b771fff 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2119,6 +2119,12 @@ Py_FinalizeEx(void) } #endif /* Py_TRACE_REFS */ +#ifdef WITH_PYMALLOC + if (malloc_stats) { + _PyObject_DebugMallocStats(stderr); + } +#endif + finalize_interp_delete(tstate->interp); #ifdef Py_REF_DEBUG @@ -2129,12 +2135,6 @@ Py_FinalizeEx(void) #endif _Py_FinalizeAllocatedBlocks(runtime); -#ifdef WITH_PYMALLOC - if (malloc_stats) { - _PyObject_DebugMallocStats(stderr); - } -#endif - call_ll_exitfuncs(runtime); _PyRuntime_Finalize(); From 9e052619a6d32051394444c24d3185db1735a893 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Tue, 4 Jun 2024 18:22:22 +0800 Subject: [PATCH 363/903] Fix typos in documentation and comments (#119763) --- Python/brc.c | 4 ++-- Python/ceval.c | 2 +- Python/flowgraph.c | 2 +- Python/gc.c | 4 ++-- Python/import.c | 2 +- Python/optimizer.c | 2 +- Python/vm-state.md | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Python/brc.c b/Python/brc.c index 8f87bc33007bcf..d27687052aec19 100644 --- a/Python/brc.c +++ b/Python/brc.c @@ -14,7 +14,7 @@ // thread states within each bucket. // // The queueing thread uses the eval breaker mechanism to notify the owning -// thread that it has objects to merge. Additionaly, all queued objects are +// thread that it has objects to merge. Additionally, all queued objects are // merged during GC. #include "Python.h" #include "pycore_object.h" // _Py_ExplicitMergeRefcount @@ -197,7 +197,7 @@ _Py_brc_after_fork(PyInterpreterState *interp) { // Unlock all bucket mutexes. Some of the buckets may be locked because // locks can be handed off to a parked thread (see lock.c). We don't have - // to worry about consistency here, becuase no thread can be actively + // to worry about consistency here, because no thread can be actively // modifying a bucket, but it might be paused (not yet woken up) on a // PyMutex_Lock while holding that lock. for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) { diff --git a/Python/ceval.c b/Python/ceval.c index 324d062fe9bb43..e3968b07486463 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1478,7 +1478,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, localsplus[total_args] = u; } else if (argcount > n) { - /* Too many postional args. Error is reported later */ + /* Too many positional args. Error is reported later */ for (j = n; j < argcount; j++) { Py_DECREF(args[j]); } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index b0c8004130fb07..17617e119fdaa4 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2861,7 +2861,7 @@ _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, } /* This is used by _PyCompile_Assemble to fill in the jump and exception - * targets in a synthetic CFG (which is not the ouptut of the builtin compiler). + * targets in a synthetic CFG (which is not the output of the builtin compiler). */ int _PyCfg_JumpLabelsToTargets(cfg_builder *g) diff --git a/Python/gc.c b/Python/gc.c index aa8b216124c36a..b87697e1e5ecfd 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1,5 +1,5 @@ // This implements the reference cycle garbage collector. -// The Python module inteface to the collector is in gcmodule.c. +// The Python module interface to the collector is in gcmodule.c. // See https://devguide.python.org/internals/garbage-collector/ #include "Python.h" @@ -1260,7 +1260,7 @@ gc_list_set_space(PyGC_Head *list, int space) * the incremental collector must progress through the old * space faster than objects are added to the old space. * - * Each young or incremental collection adds a numebr of + * Each young or incremental collection adds a number of * objects, S (for survivors) to the old space, and * incremental collectors scan I objects from the old space. * I > S must be true. We also want I > S * N to be where diff --git a/Python/import.c b/Python/import.c index 6fe6df4db4f55e..351d463dcab465 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1961,7 +1961,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, * * However, for single-phase init the module's init function will * create the module, create other objects (and allocate other - * memory), populate it and its module state, and initialze static + * memory), populate it and its module state, and initialize static * types. Some modules store other objects and data in global C * variables and register callbacks with the runtime/stdlib or * even external libraries (which is part of why we can't just diff --git a/Python/optimizer.c b/Python/optimizer.c index 5b4a6ff8cb3dad..4dc3438b6c23a4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1455,7 +1455,7 @@ PyUnstable_Optimizer_NewCounter(void) /* We use a bloomfilter with k = 6, m = 256 * The choice of k and the following constants - * could do with a more rigourous analysis, + * could do with a more rigorous analysis, * but here is a simple analysis: * * We want to keep the false positive rate low. diff --git a/Python/vm-state.md b/Python/vm-state.md index 4c68ba3b575cc8..b3246557dbeea3 100644 --- a/Python/vm-state.md +++ b/Python/vm-state.md @@ -87,4 +87,4 @@ Tier 2 IR entries are all the same size; there is no equivalent to `EXTENDED_ARG - **opcode**: Sometimes the same as a Tier 1 opcode, sometimes a separate micro opcode. Tier 2 opcodes are 9 bits (as opposed to Tier 1 opcodes, which fit in 8 bits). By convention, Tier 2 opcode names start with `_`. - **oparg**: The argument. Usually the same as the Tier 1 oparg after expansion of `EXTENDED_ARG` prefixes. Up to 32 bits. -- **operand**: An aditional argument, Typically the value of *one* cache item from the Tier 1 inline cache, up to 64 bits. +- **operand**: An additional argument, Typically the value of *one* cache item from the Tier 1 inline cache, up to 64 bits. From dce14bb2dce7887df40ae5c13b0d13e0dafceff7 Mon Sep 17 00:00:00 2001 From: Kaundur Date: Tue, 4 Jun 2024 12:48:05 +0100 Subject: [PATCH 364/903] gh-118868: logging QueueHandler fix passing of kwargs (GH-118869) Co-authored-by: Nice Zombies Co-authored-by: Vinay Sajip --- Lib/logging/config.py | 16 +++++----- Lib/test/test_logging.py | 29 +++++++++++++++++++ ...-05-09-21-36-11.gh-issue-118868.uckxxP.rst | 2 ++ 3 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-09-21-36-11.gh-issue-118868.uckxxP.rst diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 860e4751207470..ac45d6809c805c 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -725,16 +725,16 @@ def add_filters(self, filterer, filters): def _configure_queue_handler(self, klass, **kwargs): if 'queue' in kwargs: - q = kwargs['queue'] + q = kwargs.pop('queue') else: q = queue.Queue() # unbounded - rhl = kwargs.get('respect_handler_level', False) - if 'listener' in kwargs: - lklass = kwargs['listener'] - else: - lklass = logging.handlers.QueueListener - listener = lklass(q, *kwargs.get('handlers', []), respect_handler_level=rhl) - handler = klass(q) + + rhl = kwargs.pop('respect_handler_level', False) + lklass = kwargs.pop('listener', logging.handlers.QueueListener) + handlers = kwargs.pop('handlers', []) + + listener = lklass(q, *handlers, respect_handler_level=rhl) + handler = klass(q, **kwargs) handler.listener = listener return handler diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 97d7c9fb167ec1..9ebd3457a18d68 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3976,6 +3976,35 @@ def test_111615(self): } logging.config.dictConfig(config) + # gh-118868: check if kwargs are passed to logging QueueHandler + def test_kwargs_passing(self): + class CustomQueueHandler(logging.handlers.QueueHandler): + def __init__(self, *args, **kwargs): + super().__init__(queue.Queue()) + self.custom_kwargs = kwargs + + custom_kwargs = {'foo': 'bar'} + + config = { + 'version': 1, + 'handlers': { + 'custom': { + 'class': CustomQueueHandler, + **custom_kwargs + }, + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['custom'] + } + } + + logging.config.dictConfig(config) + + handler = logging.getHandlerByName('custom') + self.assertEqual(handler.custom_kwargs, custom_kwargs) + + class ManagerTest(BaseTest): def test_manager_loggerclass(self): logged = [] diff --git a/Misc/NEWS.d/next/Library/2024-05-09-21-36-11.gh-issue-118868.uckxxP.rst b/Misc/NEWS.d/next/Library/2024-05-09-21-36-11.gh-issue-118868.uckxxP.rst new file mode 100644 index 00000000000000..372a809d9594b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-21-36-11.gh-issue-118868.uckxxP.rst @@ -0,0 +1,2 @@ +Fixed issue where kwargs were no longer passed to the logging handler +QueueHandler From 99d945c0c006e3246ac00338e37c443c6e08fc5c Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Tue, 4 Jun 2024 13:20:50 +0100 Subject: [PATCH 365/903] =?UTF-8?q?gh-119819:=20Fix=20regression=20to=20al?= =?UTF-8?q?low=20logging=20configuration=20with=20multipr=E2=80=A6=20(GH-1?= =?UTF-8?q?20030)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/logging/config.py | 4 ++- Lib/test/test_logging.py | 26 +++++++++++++++++++ ...-06-04-12-23-01.gh-issue-119819.WKKrYh.rst | 2 ++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-12-23-01.gh-issue-119819.WKKrYh.rst diff --git a/Lib/logging/config.py b/Lib/logging/config.py index ac45d6809c805c..0b10bf82b60a36 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -781,8 +781,10 @@ def configure_handler(self, config): # raise ValueError('No handlers specified for a QueueHandler') if 'queue' in config: from multiprocessing.queues import Queue as MPQueue + from multiprocessing import Manager as MM + proxy_queue = MM().Queue() qspec = config['queue'] - if not isinstance(qspec, (queue.Queue, MPQueue)): + if not isinstance(qspec, (queue.Queue, MPQueue, type(proxy_queue))): if isinstance(qspec, str): q = self.resolve(qspec) if not callable(q): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 9ebd3457a18d68..d3e5ac2be2e21e 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3926,6 +3926,32 @@ def test_config_queue_handler(self): msg = str(ctx.exception) self.assertEqual(msg, "Unable to configure handler 'ah'") + @unittest.skipIf(support.is_wasi, "WASI does not have multiprocessing.") + def test_multiprocessing_queues(self): + # See gh-119819 + cd = copy.deepcopy(self.config_queue_handler) + from multiprocessing import Queue as MQ, Manager as MM + q1 = MQ() # this can't be pickled + q2 = MM().Queue() # a proxy queue for use when pickling is needed + for qspec in (q1, q2): + fn = make_temp_file('.log', 'test_logging-cmpqh-') + cd['handlers']['h1']['filename'] = fn + cd['handlers']['ah']['queue'] = qspec + qh = None + try: + self.apply_config(cd) + qh = logging.getHandlerByName('ah') + self.assertEqual(sorted(logging.getHandlerNames()), ['ah', 'h1']) + self.assertIsNotNone(qh.listener) + self.assertIs(qh.queue, qspec) + self.assertIs(qh.listener.queue, qspec) + finally: + h = logging.getHandlerByName('h1') + if h: + self.addCleanup(closeFileHandler, h, fn) + else: + self.addCleanup(os.remove, fn) + def test_90195(self): # See gh-90195 config = { diff --git a/Misc/NEWS.d/next/Library/2024-06-04-12-23-01.gh-issue-119819.WKKrYh.rst b/Misc/NEWS.d/next/Library/2024-06-04-12-23-01.gh-issue-119819.WKKrYh.rst new file mode 100644 index 00000000000000..f9e49c00f671f2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-12-23-01.gh-issue-119819.WKKrYh.rst @@ -0,0 +1,2 @@ +Fix regression to allow logging configuration with multiprocessing queue +types. From bd8c1f97e1709b5e8b07c31b1bc7b73acc76169d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mark=20Jason=20Dominus=20=28=E9=99=B6=E6=95=8F=E4=BF=AE=29?= Date: Tue, 4 Jun 2024 08:59:56 -0400 Subject: [PATCH 366/903] gh-94808: Reorganize _make_posargs and mark unused code (GH-119227) * Reorganize four-way if-elsif-elsif-elsif as nested if-elses * Mark unused branch in _make_posargs `names_with_default` is never `NULL`, even if there are no names with defaults. In that case it points to a structure with `size` zero. Rather than eliminating the branch, we leave it behind with an `assert(0)` in case a future change to the grammar exercises the branch. --- Parser/action_helpers.c | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 3f6c282ffa7a68..91b7e2f1058423 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -543,22 +543,30 @@ _make_posargs(Parser *p, asdl_arg_seq *plain_names, asdl_seq *names_with_default, asdl_arg_seq **posargs) { - if (plain_names != NULL && names_with_default != NULL) { - asdl_arg_seq *names_with_default_names = _get_names(p, names_with_default); - if (!names_with_default_names) { - return -1; + + if (names_with_default != NULL) { + if (plain_names != NULL) { + asdl_arg_seq *names_with_default_names = _get_names(p, names_with_default); + if (!names_with_default_names) { + return -1; + } + *posargs = (asdl_arg_seq*)_PyPegen_join_sequences( + p,(asdl_seq*)plain_names, (asdl_seq*)names_with_default_names); + } + else { + *posargs = _get_names(p, names_with_default); } - *posargs = (asdl_arg_seq*)_PyPegen_join_sequences( - p,(asdl_seq*)plain_names, (asdl_seq*)names_with_default_names); - } - else if (plain_names == NULL && names_with_default != NULL) { - *posargs = _get_names(p, names_with_default); - } - else if (plain_names != NULL && names_with_default == NULL) { - *posargs = plain_names; } else { - *posargs = _Py_asdl_arg_seq_new(0, p->arena); + if (plain_names != NULL) { + // With the current grammar, we never get here. + // If that has changed, remove the assert, and test thoroughly. + assert(0); + *posargs = plain_names; + } + else { + *posargs = _Py_asdl_arg_seq_new(0, p->arena); + } } return *posargs == NULL ? -1 : 0; } From e69d068ad0bd6a25434ea476a647b635da4d82bb Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 4 Jun 2024 09:42:13 -0400 Subject: [PATCH 367/903] gh-117657: Fix race involving GC and heap initialization (#119923) The `_PyThreadState_Bind()` function is called before the first `PyEval_AcquireThread()` so it's not synchronized with the stop the world GC. We had a race where `gc_visit_heaps()` might visit a thread's heap while it's being initialized. Use a simple atomic int to avoid visiting heaps for threads that are not yet fully initialized (i.e., before `tstate_mimalloc_bind()` is called). The race was reproducible by running: `python Lib/test/test_importlib/partial/pool_in_threads.py`. --- Include/internal/pycore_mimalloc.h | 1 + Python/gc_free_threading.c | 4 ++++ Python/pystate.c | 2 ++ Tools/tsan/suppressions_free_threading.txt | 3 --- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index 100f78d53021ee..d10b01d5b49b19 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -52,6 +52,7 @@ struct _mimalloc_thread_state { mi_heap_t *current_object_heap; mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT]; mi_tld_t tld; + int initialized; struct llist_node page_list; }; #endif diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index d005b79ff40dbf..f19362c9573812 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -252,6 +252,10 @@ gc_visit_heaps_lock_held(PyInterpreterState *interp, mi_block_visit_fun *visitor // visit each thread's heaps for GC objects for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { struct _mimalloc_thread_state *m = &((_PyThreadStateImpl *)p)->mimalloc; + if (!_Py_atomic_load_int(&m->initialized)) { + // The thread may not have called tstate_mimalloc_bind() yet. + continue; + } arg->offset = offset_base; if (!mi_heap_visit_blocks(&m->heaps[_Py_MIMALLOC_HEAP_GC], true, diff --git a/Python/pystate.c b/Python/pystate.c index d0293915db7689..e1a95907b57d20 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -3074,6 +3074,8 @@ tstate_mimalloc_bind(PyThreadState *tstate) // _PyObject_GC_New() and similar functions temporarily override this to // use one of the GC heaps. mts->current_object_heap = &mts->heaps[_Py_MIMALLOC_HEAP_OBJECT]; + + _Py_atomic_store_int(&mts->initialized, 1); #endif } diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index d5fcac61f0db04..8b64d1ff321858 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -25,7 +25,6 @@ race:free_threadstate race_top:_add_to_weak_set race_top:_in_weak_set -race_top:_mi_heap_delayed_free_partial race_top:_PyEval_EvalFrameDefault race_top:_PyImport_AcquireLock race_top:_PyImport_ReleaseLock @@ -33,7 +32,6 @@ race_top:_PyType_HasFeature race_top:assign_version_tag race_top:insertdict race_top:lookup_tp_dict -race_top:mi_heap_visit_pages race_top:PyMember_GetOne race_top:PyMember_SetOne race_top:new_reference @@ -58,7 +56,6 @@ race_top:_Py_slot_tp_getattr_hook race_top:add_threadstate race_top:dump_traceback race_top:fatal_error -race_top:mi_page_decode_padding race_top:_multiprocessing_SemLock_release_impl race_top:_PyFrame_GetCode race_top:_PyFrame_Initialize From ff1857d6ed52fab8ef1507c289d89ee545ca6478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 4 Jun 2024 16:24:22 +0200 Subject: [PATCH 368/903] gh-120029: export `DEF_TYPE_PARAM` compiler flag (#120028) --- Doc/library/symtable.rst | 4 ++++ Lib/symtable.py | 17 +++++++++++++---- Lib/test/test_symtable.py | 2 ++ ...24-06-04-14-54-46.gh-issue-120029._1YdTf.rst | 2 ++ Modules/symtablemodule.c | 4 +++- 5 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-14-54-46.gh-issue-120029._1YdTf.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 0480502158433a..e17a33f7feb1ab 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -151,6 +151,10 @@ Examining Symbol Tables Return ``True`` if the symbol is a parameter. + .. method:: is_type_parameter() + + Return ``True`` if the symbol is a type parameter. + .. method:: is_global() Return ``True`` if the symbol is global. diff --git a/Lib/symtable.py b/Lib/symtable.py index 17f820abd56660..ba2f0dafcd0063 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -1,9 +1,13 @@ """Interface to the compiler's internal symbol tables""" import _symtable -from _symtable import (USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM, - DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE, - LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL) +from _symtable import ( + USE, + DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, + DEF_PARAM, DEF_TYPE_PARAM, DEF_IMPORT, DEF_BOUND, DEF_ANNOT, + SCOPE_OFF, SCOPE_MASK, + FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL +) import weakref @@ -253,13 +257,18 @@ def is_referenced(self): """Return *True* if the symbol is used in its block. """ - return bool(self.__flags & _symtable.USE) + return bool(self.__flags & USE) def is_parameter(self): """Return *True* if the symbol is a parameter. """ return bool(self.__flags & DEF_PARAM) + def is_type_parameter(self): + """Return *True* if the symbol is a type parameter. + """ + return bool(self.__flags & DEF_TYPE_PARAM) + def is_global(self): """Return *True* if the symbol is global. """ diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 92b78a8086a83d..ef2a228b15ed4e 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -299,6 +299,8 @@ def test_symbol_repr(self): "") self.assertEqual(repr(self.other_internal.lookup("some_var")), "") + self.assertEqual(repr(self.GenericMine.lookup("T")), + "") def test_symtable_entry_repr(self): expected = f"" diff --git a/Misc/NEWS.d/next/Library/2024-06-04-14-54-46.gh-issue-120029._1YdTf.rst b/Misc/NEWS.d/next/Library/2024-06-04-14-54-46.gh-issue-120029._1YdTf.rst new file mode 100644 index 00000000000000..e8ea1077139f71 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-14-54-46.gh-issue-120029._1YdTf.rst @@ -0,0 +1,2 @@ +Expose :meth:`symtable.Symbol.is_type_parameter` in the :mod:`symtable` +module. Patch by Bénédikt Tran. diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index b4dbb54c3b47b0..63c4dd4225298d 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -75,6 +75,7 @@ symtable_init_constants(PyObject *m) if (PyModule_AddIntMacro(m, DEF_NONLOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_LOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_PARAM) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_TYPE_PARAM) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_FREE) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_FREE_CLASS) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_IMPORT) < 0) return -1; @@ -83,7 +84,8 @@ symtable_init_constants(PyObject *m) if (PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock) < 0) return -1; - if (PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock) < 0) return -1; + if (PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock) < 0) + return -1; if (PyModule_AddIntConstant(m, "TYPE_MODULE", ModuleBlock) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_ANNOTATION", AnnotationBlock) < 0) From 4dcd91ceafce91ec37bb1a9d544e41fc65578994 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 4 Jun 2024 11:20:01 -0400 Subject: [PATCH 369/903] gh-119588: Update docs to reflect decision to include the change with Python 3.13 and not 3.12. (#120043) --- Doc/library/zipfile.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index a4d9a1852f8f0d..5583c6b24be5c6 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -591,8 +591,8 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. .. versionadded:: 3.12 - .. versionchanged:: 3.12.4 - Prior to 3.12.4, ``is_symlink`` would unconditionally return ``False``. + .. versionchanged:: 3.13 + Previously, ``is_symlink`` would unconditionally return ``False``. .. method:: Path.exists() From 8fc7653766b106bdbc4ff6154e0020aea4ab15e6 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 4 Jun 2024 18:09:31 +0200 Subject: [PATCH 370/903] gh-120041: Do not use append_to_screen when completions are visible (GH-120042) --- Lib/_pyrepl/commands.py | 7 +++++- Lib/_pyrepl/completing_reader.py | 20 +++++++++------- Lib/test/test_pyrepl/support.py | 2 +- Lib/test/test_pyrepl/test_reader.py | 37 ++++++++++++++++++++++++++++- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 2ef5dada9d9e58..b967f5206614f8 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -365,7 +365,12 @@ def do(self) -> None: r = self.reader text = self.event * r.get_arg() r.insert(text) - if len(text) == 1 and r.pos == len(r.buffer): + if ( + len(text) == 1 and + r.pos == len(r.buffer) and + not r.cmpltn_menu_visible and # type: ignore[attr-defined] + not r.cmpltn_message_visible # type: ignore[attr-defined] + ): r.calc_screen = r.append_to_screen diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index c11d2dabdd2792..215ad8753c9f8b 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -187,18 +187,20 @@ def do(self) -> None: if p: r.insert(p) if last_is_completer: - if not r.cmpltn_menu_visible: - r.cmpltn_menu_visible = True + r.cmpltn_menu_visible = True + r.cmpltn_message_visible = False r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, r.cmpltn_menu_end, r.use_brackets, r.sort_in_column) r.dirty = True - elif stem + p in completions: - r.msg = "[ complete but not unique ]" - r.dirty = True - else: - r.msg = "[ not unique ]" - r.dirty = True + elif not r.cmpltn_menu_visible: + r.cmpltn_message_visible = True + if stem + p in completions: + r.msg = "[ complete but not unique ]" + r.dirty = True + else: + r.msg = "[ not unique ]" + r.dirty = True class self_insert(commands.self_insert): @@ -236,6 +238,7 @@ class CompletingReader(Reader): ### Instance variables cmpltn_menu: list[str] = field(init=False) cmpltn_menu_visible: bool = field(init=False) + cmpltn_message_visible: bool = field(init=False) cmpltn_menu_end: int = field(init=False) cmpltn_menu_choices: list[str] = field(init=False) @@ -271,6 +274,7 @@ def finish(self) -> None: def cmpltn_reset(self) -> None: self.cmpltn_menu = [] self.cmpltn_menu_visible = False + self.cmpltn_message_visible = False self.cmpltn_menu_end = 0 self.cmpltn_menu_choices = [] diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py index d2f5429aea7a11..e807b5f3404550 100644 --- a/Lib/test/test_pyrepl/support.py +++ b/Lib/test/test_pyrepl/support.py @@ -39,7 +39,7 @@ def code_to_events(code: str): def prepare_reader(console: Console, **kwargs): - config = ReadlineConfig(readline_completer=None) + config = ReadlineConfig(readline_completer=kwargs.pop("readline_completer", None)) reader = ReadlineAlikeReader(console=console, config=config) reader.more_lines = partial(more_lines, namespace=None) reader.paste_mode = True # Avoid extra indents diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 9fb956b655594f..d02815bfa11d74 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -1,5 +1,6 @@ import itertools import functools +import rlcompleter from unittest import TestCase from .support import handle_all_events, handle_events_narrow_console, code_to_events, prepare_reader @@ -9,7 +10,7 @@ class TestReader(TestCase): def assert_screen_equals(self, reader, expected): - actual = reader.calc_screen() + actual = reader.screen expected = expected.split("\n") self.assertListEqual(actual, expected) @@ -208,3 +209,37 @@ def test_prompt_length(self): prompt, l = Reader.process_prompt(ps1) self.assertEqual(prompt, "\033[0;32m樂>\033[0m> ") self.assertEqual(l, 5) + + def test_completions_updated_on_key_press(self): + namespace = {"itertools": itertools} + code = "itertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), # Two tabs for completion + Event(evt='key', data='\t', raw=bytearray(b'\t')), + ], code_to_events("a")) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + + actual = reader.screen + self.assertEqual(len(actual), 2) + self.assertEqual(actual[0].rstrip(), "itertools.accumulate(") + self.assertEqual(actual[1], f"{code}a") + + def test_key_press_on_tab_press_once(self): + namespace = {"itertools": itertools} + code = "itertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), + ], code_to_events("a")) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + + self.assert_screen_equals(reader, f"{code}a") From 5f03f0913413ecc4942367cf62ce3a5a5b5d84a5 Mon Sep 17 00:00:00 2001 From: Trey Hunner Date: Tue, 4 Jun 2024 09:28:08 -0700 Subject: [PATCH 371/903] Fix incorrect pull GitHub link in What's New (#120045) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.13.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 903de3c04b4a07..dfbeadce0eea27 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1006,7 +1006,7 @@ random ------ * Add a :ref:`command-line interface `. - (Contributed by Hugo van Kemenade in :gh:`54321`.) + (Contributed by Hugo van Kemenade in :gh:`118131`.) re -- From 7111d9605f9db7aa0b095bb8ece7ccc0b8115c3f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 4 Jun 2024 19:36:37 +0300 Subject: [PATCH 372/903] gh-89928: Fix integer conversion of device numbers (GH-31794) Fix os.major(), os.minor() and os.makedev(). Support device numbers larger than 2**63-1. Support non-existent device number (NODEV). --- Lib/test/test_posix.py | 15 +++- .../2022-03-10-16-47-57.bpo-45767.ywmyo1.rst | 3 + Modules/clinic/posixmodule.c.h | 32 +++---- Modules/posixmodule.c | 88 +++++++++++++++---- 4 files changed, 99 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-03-10-16-47-57.bpo-45767.ywmyo1.rst diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 7e5f04c22bd6d3..908354cb8574d1 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -704,7 +704,8 @@ def test_makedev(self): self.assertEqual(posix.major(dev), major) self.assertRaises(TypeError, posix.major, float(dev)) self.assertRaises(TypeError, posix.major) - self.assertRaises((ValueError, OverflowError), posix.major, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.major, x) minor = posix.minor(dev) self.assertIsInstance(minor, int) @@ -712,13 +713,23 @@ def test_makedev(self): self.assertEqual(posix.minor(dev), minor) self.assertRaises(TypeError, posix.minor, float(dev)) self.assertRaises(TypeError, posix.minor) - self.assertRaises((ValueError, OverflowError), posix.minor, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.minor, x) self.assertEqual(posix.makedev(major, minor), dev) self.assertRaises(TypeError, posix.makedev, float(major), minor) self.assertRaises(TypeError, posix.makedev, major, float(minor)) self.assertRaises(TypeError, posix.makedev, major) self.assertRaises(TypeError, posix.makedev) + for x in -2, 2**32, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.makedev, x, minor) + self.assertRaises((ValueError, OverflowError), posix.makedev, major, x) + + if sys.platform == 'linux': + NODEV = -1 + self.assertEqual(posix.major(NODEV), NODEV) + self.assertEqual(posix.minor(NODEV), NODEV) + self.assertEqual(posix.makedev(NODEV, NODEV), NODEV) def _test_all_chown_common(self, chown_func, first_param, stat_func): """Common code for chown, fchown and lchown tests.""" diff --git a/Misc/NEWS.d/next/Library/2022-03-10-16-47-57.bpo-45767.ywmyo1.rst b/Misc/NEWS.d/next/Library/2022-03-10-16-47-57.bpo-45767.ywmyo1.rst new file mode 100644 index 00000000000000..0cdf1e84157777 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-03-10-16-47-57.bpo-45767.ywmyo1.rst @@ -0,0 +1,3 @@ +Fix integer conversion in :func:`os.major`, :func:`os.minor`, and +:func:`os.makedev`. Support device numbers larger than ``2**63-1``. Support +non-existent device number (``NODEV``). diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index c7a447b455c594..83dcc7a60c2110 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -8685,7 +8685,7 @@ PyDoc_STRVAR(os_major__doc__, #define OS_MAJOR_METHODDEF \ {"major", (PyCFunction)os_major, METH_O, os_major__doc__}, -static unsigned int +static PyObject * os_major_impl(PyObject *module, dev_t device); static PyObject * @@ -8693,16 +8693,11 @@ os_major(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; dev_t device; - unsigned int _return_value; if (!_Py_Dev_Converter(arg, &device)) { goto exit; } - _return_value = os_major_impl(module, device); - if ((_return_value == (unsigned int)-1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromUnsignedLong((unsigned long)_return_value); + return_value = os_major_impl(module, device); exit: return return_value; @@ -8721,7 +8716,7 @@ PyDoc_STRVAR(os_minor__doc__, #define OS_MINOR_METHODDEF \ {"minor", (PyCFunction)os_minor, METH_O, os_minor__doc__}, -static unsigned int +static PyObject * os_minor_impl(PyObject *module, dev_t device); static PyObject * @@ -8729,16 +8724,11 @@ os_minor(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; dev_t device; - unsigned int _return_value; if (!_Py_Dev_Converter(arg, &device)) { goto exit; } - _return_value = os_minor_impl(module, device); - if ((_return_value == (unsigned int)-1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromUnsignedLong((unsigned long)_return_value); + return_value = os_minor_impl(module, device); exit: return return_value; @@ -8758,25 +8748,23 @@ PyDoc_STRVAR(os_makedev__doc__, {"makedev", _PyCFunction_CAST(os_makedev), METH_FASTCALL, os_makedev__doc__}, static dev_t -os_makedev_impl(PyObject *module, int major, int minor); +os_makedev_impl(PyObject *module, dev_t major, dev_t minor); static PyObject * os_makedev(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int major; - int minor; + dev_t major; + dev_t minor; dev_t _return_value; if (!_PyArg_CheckPositional("makedev", nargs, 2, 2)) { goto exit; } - major = PyLong_AsInt(args[0]); - if (major == -1 && PyErr_Occurred()) { + if (!_Py_Dev_Converter(args[0], &major)) { goto exit; } - minor = PyLong_AsInt(args[1]); - if (minor == -1 && PyErr_Occurred()) { + if (!_Py_Dev_Converter(args[1], &minor)) { goto exit; } _return_value = os_makedev_impl(module, major, minor); @@ -12795,4 +12783,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=300bd1c54dc43765 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=49c2d7a65f7a9f3b input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index bb35cfd9cdb138..1251ea63348946 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -18,6 +18,7 @@ #include "pycore_fileutils.h" // _Py_closerange() #include "pycore_import.h" // _PyImport_ReInitLock() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_LookupSpecial() #include "pycore_pylifecycle.h" // _PyOS_URandom() @@ -967,16 +968,46 @@ _Py_Gid_Converter(PyObject *obj, gid_t *p) #endif /* MS_WINDOWS */ -#define _PyLong_FromDev PyLong_FromLongLong +static PyObject * +_PyLong_FromDev(dev_t dev) +{ +#ifdef NODEV + if (dev == NODEV) { + return PyLong_FromLongLong((long long)dev); + } +#endif + return PyLong_FromUnsignedLongLong((unsigned long long)dev); +} #if (defined(HAVE_MKNOD) && defined(HAVE_MAKEDEV)) || defined(HAVE_DEVICE_MACROS) static int _Py_Dev_Converter(PyObject *obj, void *p) { - *((dev_t *)p) = PyLong_AsUnsignedLongLong(obj); - if (PyErr_Occurred()) +#ifdef NODEV + if (PyLong_Check(obj) && _PyLong_IsNegative((PyLongObject *)obj)) { + int overflow; + long long result = PyLong_AsLongLongAndOverflow(obj, &overflow); + if (result == -1 && PyErr_Occurred()) { + return 0; + } + if (!overflow && result == (long long)NODEV) { + *((dev_t *)p) = NODEV; + return 1; + } + } +#endif + + unsigned long long result = PyLong_AsUnsignedLongLong(obj); + if (result == (unsigned long long)-1 && PyErr_Occurred()) { + return 0; + } + if ((unsigned long long)(dev_t)result != result) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C dev_t"); return 0; + } + *((dev_t *)p) = (dev_t)result; return 1; } #endif /* (HAVE_MKNOD && HAVE_MAKEDEV) || HAVE_DEVICE_MACROS */ @@ -12517,9 +12548,31 @@ os_mknod_impl(PyObject *module, path_t *path, int mode, dev_t device, #endif /* defined(HAVE_MKNOD) && defined(HAVE_MAKEDEV) */ +static PyObject * +major_minor_conv(unsigned int value) +{ +#ifdef NODEV + if (value == (unsigned int)NODEV) { + return PyLong_FromLong((int)NODEV); + } +#endif + return PyLong_FromUnsignedLong(value); +} + +static int +major_minor_check(dev_t value) +{ +#ifdef NODEV + if (value == NODEV) { + return 1; + } +#endif + return (dev_t)(unsigned int)value == value; +} + #ifdef HAVE_DEVICE_MACROS /*[clinic input] -os.major -> unsigned_int +os.major device: dev_t / @@ -12527,16 +12580,16 @@ os.major -> unsigned_int Extracts a device major number from a raw device number. [clinic start generated code]*/ -static unsigned int +static PyObject * os_major_impl(PyObject *module, dev_t device) -/*[clinic end generated code: output=5b3b2589bafb498e input=1e16a4d30c4d4462]*/ +/*[clinic end generated code: output=4071ffee17647891 input=b1a0a14ec9448229]*/ { - return major(device); + return major_minor_conv(major(device)); } /*[clinic input] -os.minor -> unsigned_int +os.minor device: dev_t / @@ -12544,28 +12597,33 @@ os.minor -> unsigned_int Extracts a device minor number from a raw device number. [clinic start generated code]*/ -static unsigned int +static PyObject * os_minor_impl(PyObject *module, dev_t device) -/*[clinic end generated code: output=5e1a25e630b0157d input=0842c6d23f24c65e]*/ +/*[clinic end generated code: output=306cb78e3bc5004f input=2f686e463682a9da]*/ { - return minor(device); + return major_minor_conv(minor(device)); } /*[clinic input] os.makedev -> dev_t - major: int - minor: int + major: dev_t + minor: dev_t / Composes a raw device number from the major and minor device numbers. [clinic start generated code]*/ static dev_t -os_makedev_impl(PyObject *module, int major, int minor) -/*[clinic end generated code: output=881aaa4aba6f6a52 input=4b9fd8fc73cbe48f]*/ +os_makedev_impl(PyObject *module, dev_t major, dev_t minor) +/*[clinic end generated code: output=cad6125c51f5af80 input=2146126ec02e55c1]*/ { + if (!major_minor_check(major) || !major_minor_check(minor)) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C unsigned int"); + return (dev_t)-1; + } return makedev(major, minor); } #endif /* HAVE_DEVICE_MACROS */ From e0799352823289fafb8131341abd751923ee9c08 Mon Sep 17 00:00:00 2001 From: Christopher Chavez Date: Tue, 4 Jun 2024 11:47:15 -0500 Subject: [PATCH 373/903] gh-112672: Fix builtin Tkinter with Tcl 9.0 (GH-112681) * Add declaration of Tcl_AppInit(), missing in Tcl 9.0. * Use Tcl_Size instead of int where needed. Co-authored-by: Serhiy Storchaka --- ...-06-04-19-03-25.gh-issue-112672.K2XfZH.rst | 1 + Modules/_tkinter.c | 38 ++++++++++++------- 2 files changed, 25 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-19-03-25.gh-issue-112672.K2XfZH.rst diff --git a/Misc/NEWS.d/next/Library/2024-06-04-19-03-25.gh-issue-112672.K2XfZH.rst b/Misc/NEWS.d/next/Library/2024-06-04-19-03-25.gh-issue-112672.K2XfZH.rst new file mode 100644 index 00000000000000..46345bff117b19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-19-03-25.gh-issue-112672.K2XfZH.rst @@ -0,0 +1 @@ +Support building :mod:`tkinter` with Tcl 9.0. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 0cff36dd307c39..24f87c8d34c6b2 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -69,6 +69,12 @@ Copyright (C) 1994 Steen Lumholt. #define USE_DEPRECATED_TOMMATH_API 1 #endif +// As suggested by https://core.tcl-lang.org/tcl/wiki?name=Migrating+C+extensions+to+Tcl+9 +#ifndef TCL_SIZE_MAX +typedef int Tcl_Size; +#define TCL_SIZE_MAX INT_MAX +#endif + #if !(defined(MS_WINDOWS) || defined(__CYGWIN__)) #define HAVE_CREATEFILEHANDLER #endif @@ -489,7 +495,7 @@ unicodeFromTclString(const char *s) static PyObject * unicodeFromTclObj(Tcl_Obj *value) { - int len; + Tcl_Size len; #if USE_TCL_UNICODE int byteorder = NATIVE_BYTEORDER; const Tcl_UniChar *u = Tcl_GetUnicodeFromObj(value, &len); @@ -517,6 +523,10 @@ class _tkinter.tktimertoken "TkttObject *" "&Tktt_Type_spec" /**** Tkapp Object ****/ +#if TK_MAJOR_VERSION >= 9 +int Tcl_AppInit(Tcl_Interp *); +#endif + #ifndef WITH_APPINIT int Tcl_AppInit(Tcl_Interp *interp) @@ -1142,7 +1152,7 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) } if (value->typePtr == tkapp->ByteArrayType) { - int size; + Tcl_Size size; char *data = (char*)Tcl_GetByteArrayFromObj(value, &size); return PyBytes_FromStringAndSize(data, size); } @@ -1168,8 +1178,8 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) } if (value->typePtr == tkapp->ListType) { - int size; - int i, status; + Tcl_Size i, size; + int status; PyObject *elem; Tcl_Obj *tcl_elem; @@ -1225,9 +1235,9 @@ typedef struct Tkapp_CallEvent { } Tkapp_CallEvent; static void -Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, int objc) +Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, Tcl_Size objc) { - int i; + Tcl_Size i; for (i = 0; i < objc; i++) Tcl_DecrRefCount(objv[i]); if (objv != objStore) @@ -1238,7 +1248,7 @@ Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, int objc) interpreter thread, which may or may not be the calling thread. */ static Tcl_Obj** -Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) +Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, Tcl_Size *pobjc) { Tcl_Obj **objv = objStore; Py_ssize_t objc = 0, i; @@ -1286,10 +1296,10 @@ Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) Tcl_IncrRefCount(objv[i]); } } - *pobjc = (int)objc; + *pobjc = (Tcl_Size)objc; return objv; finally: - Tkapp_CallDeallocArgs(objv, objStore, (int)objc); + Tkapp_CallDeallocArgs(objv, objStore, (Tcl_Size)objc); return NULL; } @@ -1356,7 +1366,7 @@ Tkapp_CallProc(Tcl_Event *evPtr, int flags) Tkapp_CallEvent *e = (Tkapp_CallEvent *)evPtr; Tcl_Obj *objStore[ARGSZ]; Tcl_Obj **objv; - int objc; + Tcl_Size objc; int i; ENTER_PYTHON if (e->self->trace && !Tkapp_Trace(e->self, PyTuple_Pack(1, e->args))) { @@ -1412,7 +1422,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) { Tcl_Obj *objStore[ARGSZ]; Tcl_Obj **objv = NULL; - int objc, i; + Tcl_Size objc; PyObject *res = NULL; TkappObject *self = (TkappObject*)selfptr; int flags = TCL_EVAL_DIRECT | TCL_EVAL_GLOBAL; @@ -1459,6 +1469,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) { TRACE(self, ("(O)", args)); + int i; objv = Tkapp_CallArgs(args, objStore, &objc); if (!objv) return NULL; @@ -2193,13 +2204,12 @@ _tkinter_tkapp_splitlist(TkappObject *self, PyObject *arg) /*[clinic end generated code: output=13b51d34386d36fb input=2b2e13351e3c0b53]*/ { char *list; - int argc; + Tcl_Size argc, i; const char **argv; PyObject *v; - int i; if (PyTclObject_Check(arg)) { - int objc; + Tcl_Size objc; Tcl_Obj **objv; if (Tcl_ListObjGetElements(Tkapp_Interp(self), ((PyTclObject*)arg)->value, From bf8e5e53d0c359a1f9c285d855e7a5e9b6d91375 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 4 Jun 2024 19:26:44 +0200 Subject: [PATCH 374/903] gh-120041: Refactor check for visible completion menu in completing_reader (#120055) --- Lib/_pyrepl/commands.py | 7 +------ Lib/_pyrepl/completing_reader.py | 3 +++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index b967f5206614f8..2ef5dada9d9e58 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -365,12 +365,7 @@ def do(self) -> None: r = self.reader text = self.event * r.get_arg() r.insert(text) - if ( - len(text) == 1 and - r.pos == len(r.buffer) and - not r.cmpltn_menu_visible and # type: ignore[attr-defined] - not r.cmpltn_message_visible # type: ignore[attr-defined] - ): + if len(text) == 1 and r.pos == len(r.buffer): r.calc_screen = r.append_to_screen diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 215ad8753c9f8b..8df35ccb9117b1 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -210,6 +210,9 @@ def do(self) -> None: commands.self_insert.do(self) + if r.cmpltn_menu_visible or r.cmpltn_message_visible: + r.calc_screen = r.calc_complete_screen + if r.cmpltn_menu_visible: stem = r.get_stem() if len(stem) < 1: From d419d468ff4aaf6bc673354d0ee41b273d09dd3f Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 4 Jun 2024 13:38:29 -0400 Subject: [PATCH 375/903] gh-120039: Reduce expected timeout in test_siginterrupt_off (#120047) The process is expected to time out. In the refleak builds, `support.SHORT_TIMEOUT` is often five minutes and we run the tests six times, so test_signal was taking >30 minutes. --- Lib/test/test_signal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 61fb047caf6dab..591cd4177d9f41 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -698,7 +698,7 @@ def handler(signum, frame): @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") class SiginterruptTest(unittest.TestCase): - def readpipe_interrupted(self, interrupt): + def readpipe_interrupted(self, interrupt, timeout=support.SHORT_TIMEOUT): """Perform a read during which a signal will arrive. Return True if the read is interrupted by the signal and raises an exception. Return False if it returns normally. @@ -746,7 +746,7 @@ def handler(signum, frame): # wait until the child process is loaded and has started first_line = process.stdout.readline() - stdout, stderr = process.communicate(timeout=support.SHORT_TIMEOUT) + stdout, stderr = process.communicate(timeout=timeout) except subprocess.TimeoutExpired: process.kill() return False @@ -777,7 +777,7 @@ def test_siginterrupt_off(self): # If a signal handler is installed and siginterrupt is called with # a false value for the second argument, when that signal arrives, it # does not interrupt a syscall that's in progress. - interrupted = self.readpipe_interrupted(False) + interrupted = self.readpipe_interrupted(False, timeout=2) self.assertFalse(interrupted) From 010ea93b2b888149561becefeee90826bf8a2934 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 4 Jun 2024 19:46:33 +0200 Subject: [PATCH 376/903] gh-119553: Clear reader on Ctrl-C command (GH-119801) --- Lib/_pyrepl/commands.py | 1 + Lib/test/test_pyrepl/support.py | 2 ++ Lib/test/test_pyrepl/test_reader.py | 16 ++++++++++++++++ 3 files changed, 19 insertions(+) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 2ef5dada9d9e58..e94e8c25d379c1 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -221,6 +221,7 @@ def do(self) -> None: class ctrl_c(Command): def do(self) -> None: + self.reader.finish() raise KeyboardInterrupt diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py index e807b5f3404550..70e12286f7d781 100644 --- a/Lib/test/test_pyrepl/support.py +++ b/Lib/test/test_pyrepl/support.py @@ -75,6 +75,8 @@ def handle_all_events( reader.handle1() except StopIteration: pass + except KeyboardInterrupt: + pass return reader, console diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index d02815bfa11d74..079c963d19aad5 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -179,6 +179,22 @@ def test_newline_within_block_trailing_whitespace(self): self.assert_screen_equals(reader, expected) self.assertTrue(reader.finished) + def test_keyboard_interrupt_clears_screen(self): + namespace = {"itertools": itertools} + code = "import itertools\nitertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), # Two tabs for completion + Event(evt='key', data='\t', raw=bytearray(b'\t')), + Event(evt='key', data='\x03', raw=bytearray(b'\x03')), # Ctrl-C + ]) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + self.assertEqual(reader.calc_screen(), code.split("\n")) + def test_prompt_length(self): # Handles simple ASCII prompt ps1 = ">>> " From bf5e1065f4ec2077c6ca352fc1ad940a76d1f6c9 Mon Sep 17 00:00:00 2001 From: Paulo Freitas Date: Tue, 4 Jun 2024 14:55:11 -0300 Subject: [PATCH 377/903] doc: Mention the missing reflected special methods for all binary operations (GH-119931) --- Doc/reference/expressions.rst | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 00b57effd3e1c0..872773f4d28235 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1211,7 +1211,8 @@ Raising ``0.0`` to a negative power results in a :exc:`ZeroDivisionError`. Raising a negative number to a fractional power results in a :class:`complex` number. (In earlier versions it raised a :exc:`ValueError`.) -This operation can be customized using the special :meth:`~object.__pow__` method. +This operation can be customized using the special :meth:`~object.__pow__` and +:meth:`~object.__rpow__` methods. .. _unary: @@ -1299,6 +1300,9 @@ This operation can be customized using the special :meth:`~object.__mul__` and The ``@`` (at) operator is intended to be used for matrix multiplication. No builtin Python types implement this operator. +This operation can be customized using the special :meth:`~object.__matmul__` and +:meth:`~object.__rmatmul__` methods. + .. versionadded:: 3.5 .. index:: @@ -1314,8 +1318,10 @@ integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` exception. -This operation can be customized using the special :meth:`~object.__truediv__` and -:meth:`~object.__floordiv__` methods. +The division operation can be customized using the special :meth:`~object.__truediv__` +and :meth:`~object.__rtruediv__` methods. +The floor division operation can be customized using the special +:meth:`~object.__floordiv__` and :meth:`~object.__rfloordiv__` methods. .. index:: single: modulo @@ -1340,7 +1346,8 @@ also overloaded by string objects to perform old-style string formatting (also known as interpolation). The syntax for string formatting is described in the Python Library Reference, section :ref:`old-string-formatting`. -The *modulo* operation can be customized using the special :meth:`~object.__mod__` method. +The *modulo* operation can be customized using the special :meth:`~object.__mod__` +and :meth:`~object.__rmod__` methods. The floor division operator, the modulo operator, and the :func:`divmod` function are not defined for complex numbers. Instead, convert to a floating @@ -1367,7 +1374,8 @@ This operation can be customized using the special :meth:`~object.__add__` and The ``-`` (subtraction) operator yields the difference of its arguments. The numeric arguments are first converted to a common type. -This operation can be customized using the special :meth:`~object.__sub__` method. +This operation can be customized using the special :meth:`~object.__sub__` and +:meth:`~object.__rsub__` methods. .. _shifting: @@ -1388,8 +1396,10 @@ The shifting operations have lower priority than the arithmetic operations: These operators accept integers as arguments. They shift the first argument to the left or right by the number of bits given by the second argument. -This operation can be customized using the special :meth:`~object.__lshift__` and -:meth:`~object.__rshift__` methods. +The left shift operation can be customized using the special :meth:`~object.__lshift__` +and :meth:`~object.__rlshift__` methods. +The right shift operation can be customized using the special :meth:`~object.__rshift__` +and :meth:`~object.__rrshift__` methods. .. index:: pair: exception; ValueError From d9095194dde27eaabfc0b86a11989cdb9a2acfe1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 4 Jun 2024 19:32:43 +0100 Subject: [PATCH 378/903] gh-119842: Honor PyOS_InputHook in the new REPL (GH-119843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Pablo Galindo Co-authored-by: Łukasz Langa Co-authored-by: Michael Droettboom --- Lib/_pyrepl/console.py | 12 +++++- Lib/_pyrepl/reader.py | 10 ++++- Lib/_pyrepl/unix_console.py | 22 ++++++++--- Lib/_pyrepl/windows_console.py | 22 ++++++++++- Lib/test/test_pyrepl/test_reader.py | 17 +++++++++ ...-05-31-12-06-11.gh-issue-119842.tCGVsv.rst | 1 + Modules/clinic/posixmodule.c.h | 38 ++++++++++++++++++- Modules/posixmodule.c | 33 ++++++++++++++++ 8 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-31-12-06-11.gh-issue-119842.tCGVsv.rst diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index aa0bde865825c9..a8d3f520340dcf 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -33,6 +33,7 @@ if TYPE_CHECKING: from typing import IO + from typing import Callable @dataclass @@ -134,8 +135,15 @@ def getpending(self) -> Event: ... @abstractmethod - def wait(self) -> None: - """Wait for an event.""" + def wait(self, timeout: float | None) -> bool: + """Wait for an event. The return value is True if an event is + available, False if the timeout has been reached. If timeout is + None, wait forever. The timeout is in milliseconds.""" + ... + + @property + def input_hook(self) -> Callable[[], int] | None: + """Returns the current input hook.""" ... @abstractmethod diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index f2e68ef6f3ee66..beee7764e0eb84 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -650,7 +650,15 @@ def handle1(self, block: bool = True) -> bool: self.dirty = True while True: - event = self.console.get_event(block) + input_hook = self.console.input_hook + if input_hook: + input_hook() + # We use the same timeout as in readline.c: 100ms + while not self.console.wait(100): + input_hook() + event = self.console.get_event(block=False) + else: + event = self.console.get_event(block) if not event: # can only happen if we're not blocking return False diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 4bdb02261982c3..2f73a59dd1fced 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -118,9 +118,12 @@ def __init__(self): def register(self, fd, flag): self.fd = fd - - def poll(self): # note: a 'timeout' argument would be *milliseconds* - r, w, e = select.select([self.fd], [], []) + # note: The 'timeout' argument is received as *milliseconds* + def poll(self, timeout: float | None = None) -> list[int]: + if timeout is None: + r, w, e = select.select([self.fd], [], []) + else: + r, w, e = select.select([self.fd], [], [], timeout/1000) return r poll = MinimalPoll # type: ignore[assignment] @@ -385,11 +388,11 @@ def get_event(self, block: bool = True) -> Event | None: break return self.event_queue.get() - def wait(self): + def wait(self, timeout: float | None = None) -> bool: """ Wait for events on the console. """ - self.pollob.poll() + return bool(self.pollob.poll(timeout)) def set_cursor_vis(self, visible): """ @@ -527,6 +530,15 @@ def clear(self): self.__posxy = 0, 0 self.screen = [] + @property + def input_hook(self): + try: + import posix + except ImportError: + return None + if posix._is_inputhook_installed(): + return posix._inputhook + def __enable_bracketed_paste(self) -> None: os.write(self.output_fd, b"\x1b[?2004h") diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index 2277865e3262fc..f691ca3fbb07b8 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -23,6 +23,8 @@ from multiprocessing import Value import os import sys +import time +import msvcrt from abc import ABC, abstractmethod from collections import deque @@ -202,6 +204,15 @@ def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: self.screen = screen self.move_cursor(cx, cy) + @property + def input_hook(self): + try: + import nt + except ImportError: + return None + if nt._is_inputhook_installed(): + return nt._inputhook + def __write_changed_line( self, y: int, oldline: str, newline: str, px_coord: int ) -> None: @@ -460,9 +471,16 @@ def getpending(self) -> Event: processed.""" return Event("key", "", b"") - def wait(self) -> None: + def wait(self, timeout: float | None) -> bool: """Wait for an event.""" - raise NotImplementedError("No wait support") + # Poor man's Windows select loop + start_time = time.time() + while True: + if msvcrt.kbhit(): # type: ignore[attr-defined] + return True + if timeout and time.time() - start_time > timeout: + return False + time.sleep(0.01) def repaint(self) -> None: raise NotImplementedError("No repaint support") diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 079c963d19aad5..78b11323d60a85 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -2,8 +2,10 @@ import functools import rlcompleter from unittest import TestCase +from unittest.mock import MagicMock, patch from .support import handle_all_events, handle_events_narrow_console, code_to_events, prepare_reader +from test.support import import_helper from _pyrepl.console import Event from _pyrepl.reader import Reader @@ -179,6 +181,21 @@ def test_newline_within_block_trailing_whitespace(self): self.assert_screen_equals(reader, expected) self.assertTrue(reader.finished) + def test_input_hook_is_called_if_set(self): + input_hook = MagicMock() + def _prepare_console(events): + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + console.input_hook = input_hook + return console + + events = code_to_events("a") + reader, _ = handle_all_events(events, prepare_console=_prepare_console) + + self.assertEqual(len(input_hook.mock_calls), 4) + def test_keyboard_interrupt_clears_screen(self): namespace = {"itertools": itertools} code = "import itertools\nitertools." diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-31-12-06-11.gh-issue-119842.tCGVsv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-12-06-11.gh-issue-119842.tCGVsv.rst new file mode 100644 index 00000000000000..2fcb170f6226e5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-12-06-11.gh-issue-119842.tCGVsv.rst @@ -0,0 +1 @@ +Honor :c:func:`PyOS_InputHook` in the new REPL. Patch by Pablo Galindo diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 83dcc7a60c2110..69fc178331c09c 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -12116,6 +12116,42 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(MS_WINDOWS) */ +PyDoc_STRVAR(os__inputhook__doc__, +"_inputhook($module, /)\n" +"--\n" +"\n" +"Calls PyOS_CallInputHook droppong the GIL first"); + +#define OS__INPUTHOOK_METHODDEF \ + {"_inputhook", (PyCFunction)os__inputhook, METH_NOARGS, os__inputhook__doc__}, + +static PyObject * +os__inputhook_impl(PyObject *module); + +static PyObject * +os__inputhook(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__inputhook_impl(module); +} + +PyDoc_STRVAR(os__is_inputhook_installed__doc__, +"_is_inputhook_installed($module, /)\n" +"--\n" +"\n" +"Checks if PyOS_CallInputHook is set"); + +#define OS__IS_INPUTHOOK_INSTALLED_METHODDEF \ + {"_is_inputhook_installed", (PyCFunction)os__is_inputhook_installed, METH_NOARGS, os__is_inputhook_installed__doc__}, + +static PyObject * +os__is_inputhook_installed_impl(PyObject *module); + +static PyObject * +os__is_inputhook_installed(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__is_inputhook_installed_impl(module); +} + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -12783,4 +12819,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=49c2d7a65f7a9f3b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=faaa5e5ffb7b165d input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 1251ea63348946..386e942d53f539 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16784,6 +16784,37 @@ os__supports_virtual_terminal_impl(PyObject *module) } #endif +/*[clinic input] +os._inputhook + +Calls PyOS_CallInputHook droppong the GIL first +[clinic start generated code]*/ + +static PyObject * +os__inputhook_impl(PyObject *module) +/*[clinic end generated code: output=525aca4ef3c6149f input=fc531701930d064f]*/ +{ + int result = 0; + if (PyOS_InputHook) { + Py_BEGIN_ALLOW_THREADS; + result = PyOS_InputHook(); + Py_END_ALLOW_THREADS; + } + return PyLong_FromLong(result); +} + +/*[clinic input] +os._is_inputhook_installed + +Checks if PyOS_CallInputHook is set +[clinic start generated code]*/ + +static PyObject * +os__is_inputhook_installed_impl(PyObject *module) +/*[clinic end generated code: output=3b3eab4f672c689a input=ff177c9938dd76d8]*/ +{ + return PyBool_FromLong(PyOS_InputHook != NULL); +} static PyMethodDef posix_methods[] = { @@ -16997,6 +17028,8 @@ static PyMethodDef posix_methods[] = { OS__PATH_LEXISTS_METHODDEF OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF + OS__INPUTHOOK_METHODDEF + OS__IS_INPUTHOOK_INSTALLED_METHODDEF {NULL, NULL} /* Sentinel */ }; From 710cbea6604d27c7d59ae4953bf522b997a82cc7 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 4 Jun 2024 14:59:23 -0400 Subject: [PATCH 379/903] gh-120048: Make `test_imaplib` faster (#120050) The `test_imaplib` was taking 40+ minutes in the refleak build bots because the tests waiting on a client `self._setup()` was creating a client that prevented progress until its connection timed out, which scaled with the global timeout. We should set `connect=False` for the tests that don't want `_setup()` to create a client. Co-authored-by: Serhiy Storchaka --- Lib/test/test_imaplib.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 79bf7dbdbb81a0..b5384b59463742 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -458,18 +458,14 @@ def test_simple_with_statement(self): with self.imap_class(*server.server_address): pass - @requires_resource('walltime') def test_imaplib_timeout_test(self): - _, server = self._setup(SimpleIMAPHandler) - addr = server.server_address[1] - client = self.imap_class("localhost", addr, timeout=None) - self.assertEqual(client.sock.timeout, None) - client.shutdown() - client = self.imap_class("localhost", addr, timeout=support.LOOPBACK_TIMEOUT) - self.assertEqual(client.sock.timeout, support.LOOPBACK_TIMEOUT) - client.shutdown() + _, server = self._setup(SimpleIMAPHandler, connect=False) + with self.imap_class(*server.server_address, timeout=None) as client: + self.assertEqual(client.sock.timeout, None) + with self.imap_class(*server.server_address, timeout=support.LOOPBACK_TIMEOUT) as client: + self.assertEqual(client.sock.timeout, support.LOOPBACK_TIMEOUT) with self.assertRaises(ValueError): - client = self.imap_class("localhost", addr, timeout=0) + self.imap_class(*server.server_address, timeout=0) def test_imaplib_timeout_functionality_test(self): class TimeoutHandler(SimpleIMAPHandler): @@ -552,7 +548,6 @@ class NewIMAPSSLTests(NewIMAPTestsMixin, unittest.TestCase): imap_class = IMAP4_SSL server_class = SecureTCPServer - @requires_resource('walltime') def test_ssl_raises(self): ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) self.assertEqual(ssl_context.verify_mode, ssl.CERT_REQUIRED) @@ -566,17 +561,16 @@ def test_ssl_raises(self): CERTIFICATE_VERIFY_FAILED # AWS-LC )""", re.X) with self.assertRaisesRegex(ssl.CertificateError, regex): - _, server = self._setup(SimpleIMAPHandler) + _, server = self._setup(SimpleIMAPHandler, connect=False) client = self.imap_class(*server.server_address, ssl_context=ssl_context) client.shutdown() - @requires_resource('walltime') def test_ssl_verified(self): ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ssl_context.load_verify_locations(CAFILE) - _, server = self._setup(SimpleIMAPHandler) + _, server = self._setup(SimpleIMAPHandler, connect=False) client = self.imap_class("localhost", server.server_address[1], ssl_context=ssl_context) client.shutdown() From 109e1082ea92f89d42cd70f2cc7ca6fba6be9bab Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Tue, 4 Jun 2024 20:16:43 +0100 Subject: [PATCH 380/903] gh-119819: Update test to skip if _multiprocessing is unavailable. (GH-120067) --- Lib/test/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index d3e5ac2be2e21e..0c9a24e58dfd8c 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3926,9 +3926,9 @@ def test_config_queue_handler(self): msg = str(ctx.exception) self.assertEqual(msg, "Unable to configure handler 'ah'") - @unittest.skipIf(support.is_wasi, "WASI does not have multiprocessing.") def test_multiprocessing_queues(self): # See gh-119819 + import_helper.import_module('_multiprocessing') # will skip test if it's not available cd = copy.deepcopy(self.config_queue_handler) from multiprocessing import Queue as MQ, Manager as MM q1 = MQ() # this can't be pickled From 4055577221f5f52af329e87f31d81bb8fb02c504 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 4 Jun 2024 15:26:26 -0400 Subject: [PATCH 381/903] gh-119999: Fix potential race condition in `_Py_ExplicitMergeRefcount` (#120000) We need to write to `ob_ref_local` and `ob_tid` before `ob_ref_shared`. Once we mark `ob_ref_shared` as merged, some other thread may free the object because the caller also passes in `-1` as `extra` to give up its only reference. --- Objects/object.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Objects/object.c b/Objects/object.c index 2e9962f4651e1c..b7730475ac3768 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -401,24 +401,27 @@ Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) { assert(!_Py_IsImmortal(op)); + +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyThreadState_GET(), extra); +#endif + + // gh-119999: Write to ob_ref_local and ob_tid before merging the refcount. + Py_ssize_t local = (Py_ssize_t)op->ob_ref_local; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + Py_ssize_t refcnt; Py_ssize_t new_shared; Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); do { refcnt = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); - refcnt += (Py_ssize_t)op->ob_ref_local; + refcnt += local; refcnt += extra; new_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, &shared, new_shared)); - -#ifdef Py_REF_DEBUG - _Py_AddRefTotal(_PyThreadState_GET(), extra); -#endif - - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); - _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); return refcnt; } #endif /* Py_GIL_DISABLED */ From 69b3e8ea569faabccd74036e3d0e5ec7c0c62a20 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 4 Jun 2024 23:22:28 +0200 Subject: [PATCH 382/903] gh-119553: Fix console when pressing Ctrl-C within a multiline block (#120075) --- Lib/_pyrepl/commands.py | 2 ++ Lib/_pyrepl/simple_interact.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index e94e8c25d379c1..6bffed1bfe9327 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -216,11 +216,13 @@ def do(self) -> None: import signal self.reader.console.finish() + self.reader.finish() os.kill(os.getpid(), signal.SIGINT) class ctrl_c(Command): def do(self) -> None: + self.reader.console.finish() self.reader.finish() raise KeyboardInterrupt diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 256bbc7c6d7626..2e5698eb131684 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -149,7 +149,7 @@ def more_lines(unicodetext: str) -> bool: assert not more input_n += 1 except KeyboardInterrupt: - console.write("\nKeyboardInterrupt\n") + console.write("KeyboardInterrupt\n") console.resetbuffer() except MemoryError: console.write("\nMemoryError\n") From 770f3c1eadd3392c72fd55be47770234dd143a14 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 5 Jun 2024 00:00:02 +0100 Subject: [PATCH 383/903] gh-114616: Improve docs regarding changes to caches representation in dis (#120033) --- Doc/library/dis.rst | 7 ++++--- Doc/whatsnew/3.13.rst | 7 +++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index fda46d260bcb46..87d1bcdfaf3f1d 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -336,9 +336,10 @@ operation is being performed, so the intermediate analysis object isn't useful: Added the *show_caches* and *adaptive* parameters. .. versionchanged:: 3.13 - The *show_caches* parameter is deprecated and has no effect. The *cache_info* - field of each instruction is populated regardless of its value. - + The *show_caches* parameter is deprecated and has no effect. The iterator + generates the :class:`Instruction` instances with the *cache_info* + field populated (regardless of the value of *show_caches*) and it no longer + generates separate items for the cache entries. .. function:: findlinestarts(code) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index dfbeadce0eea27..a1d2a0d84e7581 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -706,6 +706,13 @@ dis the ``show_offsets`` parameter. (Contributed by Irit Katriel in :gh:`112137`.) +* :meth:`~dis.get_instructions` no longer represents cache entries as + separate instructions. Instead, it returns them as part of the + :class:`~dis.Instruction`, in the new *cache_info* field. The + *show_caches* argument to :meth:`~dis.get_instructions` is + deprecated and no longer has any effect. + (Contributed by Irit Katriel in :gh:`112962`.) + .. _whatsnew313-doctest: doctest From b6b0dcbfc054f581b6f78602e4c2e9474e3efe21 Mon Sep 17 00:00:00 2001 From: shurj0 <60540027+shurj0@users.noreply.github.com> Date: Wed, 5 Jun 2024 06:23:12 +0600 Subject: [PATCH 384/903] gh-120078: Fix struct_time attr typo tm_day -> tm_mday in Doc/library/time.rst (GH-120081) --- Doc/library/time.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index ef033d59d56185..4d7661715aa0af 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -617,7 +617,7 @@ Functions - range [1, 12] * - 2 - - .. attribute:: tm_day + - .. attribute:: tm_mday - range [1, 31] * - 3 From 983efcf15b2503fe0c05d5e03762385967962b33 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Wed, 5 Jun 2024 07:25:47 +0100 Subject: [PATCH 385/903] =?UTF-8?q?gh-119819:=20Update=20logging=20configu?= =?UTF-8?q?ration=20to=20support=20joinable=20multiproc=E2=80=A6=20(GH-120?= =?UTF-8?q?090)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-119819: Update logging configuration to support joinable multiprocessing manager queues. --- Lib/logging/config.py | 4 +++- Lib/test/test_logging.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 0b10bf82b60a36..9de84e527b18ac 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -783,8 +783,10 @@ def configure_handler(self, config): from multiprocessing.queues import Queue as MPQueue from multiprocessing import Manager as MM proxy_queue = MM().Queue() + proxy_joinable_queue = MM().JoinableQueue() qspec = config['queue'] - if not isinstance(qspec, (queue.Queue, MPQueue, type(proxy_queue))): + if not isinstance(qspec, (queue.Queue, MPQueue, + type(proxy_queue), type(proxy_joinable_queue))): if isinstance(qspec, str): q = self.resolve(qspec) if not callable(q): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 0c9a24e58dfd8c..ef2d4a621be962 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3928,12 +3928,16 @@ def test_config_queue_handler(self): def test_multiprocessing_queues(self): # See gh-119819 - import_helper.import_module('_multiprocessing') # will skip test if it's not available + + # will skip test if it's not available + import_helper.import_module('_multiprocessing') + cd = copy.deepcopy(self.config_queue_handler) from multiprocessing import Queue as MQ, Manager as MM q1 = MQ() # this can't be pickled q2 = MM().Queue() # a proxy queue for use when pickling is needed - for qspec in (q1, q2): + q3 = MM().JoinableQueue() # a joinable proxy queue + for qspec in (q1, q2, q3): fn = make_temp_file('.log', 'test_logging-cmpqh-') cd['handlers']['h1']['filename'] = fn cd['handlers']['ah']['queue'] = qspec From 5c02ea8bae2287a828840f5734966da23dc573dc Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 5 Jun 2024 12:56:01 +0100 Subject: [PATCH 386/903] gh-119287: clarify doc on BaseExceptionGroup.derive and link to it from contextlib.suppress (#119657) --- Doc/library/contextlib.rst | 4 +++- Doc/library/exceptions.rst | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index bad9da52d6a6ca..27cf99446e5980 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -314,7 +314,9 @@ Functions and classes provided: If the code within the :keyword:`!with` block raises a :exc:`BaseExceptionGroup`, suppressed exceptions are removed from the - group. If any exceptions in the group are not suppressed, a group containing them is re-raised. + group. Any exceptions of the group which are not suppressed are re-raised in + a new group which is created using the original group's :meth:`~BaseExceptionGroup.derive` + method. .. versionadded:: 3.4 diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 7879fb015bddfa..7910b306f143d7 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -989,7 +989,8 @@ their subgroups based on the types of the contained exceptions. Returns an exception group with the same :attr:`message`, but which wraps the exceptions in ``excs``. - This method is used by :meth:`subgroup` and :meth:`split`. A + This method is used by :meth:`subgroup` and :meth:`split`, which + are used in various contexts to break up an exception group. A subclass needs to override it in order to make :meth:`subgroup` and :meth:`split` return instances of the subclass rather than :exc:`ExceptionGroup`. From 4bba1c9e6cfeaf69302b501a4306668613db4b28 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 5 Jun 2024 09:23:29 -0400 Subject: [PATCH 387/903] gh-120065: Increase `collect_in_thread` period to 5 ms. (#120068) This matches the default GIL switch interval. It greatly speeds up the free-threaded build: previously, it spent nearly all its time in `gc.collect()`. --- Lib/test/test_weakref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 16da24d7805b56..ef2fe92cc219b6 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -82,7 +82,7 @@ def callback(self, ref): @contextlib.contextmanager -def collect_in_thread(period=0.0001): +def collect_in_thread(period=0.005): """ Ensure GC collections happen in a different thread, at a high frequency. """ From 10eac0269bce4e2ba575e5b549d3dd9a6da9349a Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 5 Jun 2024 16:28:47 +0100 Subject: [PATCH 388/903] gh-119786: add links to code in exception handling doc (#120077) --- InternalDocs/exception_handling.md | 49 ++++++++++-------------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md index 22d9c3bf7933f1..ec09e0769929fa 100644 --- a/InternalDocs/exception_handling.md +++ b/InternalDocs/exception_handling.md @@ -67,14 +67,18 @@ handler located at label `L1`. Handling Exceptions ------------------- -At runtime, when an exception occurs, the interpreter looks up -the offset of the current instruction in the exception table. If -it finds a handler, control flow transfers to it. Otherwise, the +At runtime, when an exception occurs, the interpreter calls +``get_exception_handler()`` in +[Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c) +to look up the offset of the current instruction in the exception +table. If it finds a handler, control flow transfers to it. Otherwise, the exception bubbles up to the caller, and the caller's frame is checked for a handler covering the `CALL` instruction. This repeats until a handler is found or the topmost frame is reached. If no handler is found, the program terminates. During unwinding, -the traceback is constructed as each frame is added to it. +the traceback is constructed as each frame is added to it by +``PyTraceBack_Here()``, which is in +[Python/traceback.c](https://github.com/python/cpython/blob/main/Python/traceback.c). Along with the location of an exception handler, each entry of the exception table also contains the stack depth of the `try` instruction @@ -169,33 +173,12 @@ which is then encoded as: for a total of five bytes. +The code to construct the exception table is in ``assemble_exception_table()`` +in [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c). -Script to parse the exception table ------------------------------------ - -``` -def parse_varint(iterator): - b = next(iterator) - val = b & 63 - while b&64: - val <<= 6 - b = next(iterator) - val |= b&63 - return val -``` -``` -def parse_exception_table(code): - iterator = iter(code.co_exceptiontable) - try: - while True: - start = parse_varint(iterator)*2 - length = parse_varint(iterator)*2 - end = start + length - 2 # Present as inclusive, not exclusive - target = parse_varint(iterator)*2 - dl = parse_varint(iterator) - depth = dl >> 1 - lasti = bool(dl&1) - yield start, end, target, depth, lasti - except StopIteration: - return -``` +The interpreter's function to lookup the table by instruction offset is +``get_exception_handler()`` in +[Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c). +The Python function ``_parse_exception_table()`` in +[Lib/dis.py](https://github.com/python/cpython/blob/main/Lib/dis.py) +returns the exception table content as a list of namedtuple instances. From 14e3c7071bd1add30d4b69b62e011c7d38aebd9b Mon Sep 17 00:00:00 2001 From: benchatt Date: Wed, 5 Jun 2024 10:35:40 -0700 Subject: [PATCH 389/903] gh-115225: Raise error on unsupported ISO 8601 time strings (#119339) Some time strings that contain fractional hours or minutes are permitted by ISO 8601, but such strings are very unlikely to be intentional. The current parser does not parse such strings correctly or raise an error. This change raises a ValueError when hours or minutes contain a decimal mark. Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/test/datetimetester.py | 2 ++ Misc/ACKS | 1 + .../Library/2024-05-21-19-10-30.gh-issue-115225.eRmfJH.rst | 1 + Modules/_datetimemodule.c | 3 +++ 4 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-05-21-19-10-30.gh-issue-115225.eRmfJH.rst diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 535b17d0727611..3759504b02e550 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -4412,6 +4412,8 @@ def test_fromisoformat_fails(self): '12:30:45.123456-', # Extra at end of microsecond time '12:30:45.123456+', # Extra at end of microsecond time '12:30:45.123456+12:00:30a', # Extra at end of full time + '12.5', # Decimal mark at end of hour + '12:30,5', # Decimal mark at end of minute ] for bad_str in bad_strs: diff --git a/Misc/ACKS b/Misc/ACKS index 2e7e12481bacd7..af92d81ff3141a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -315,6 +315,7 @@ Greg Chapman Mitch Chapman Matt Chaput William Chargin +Ben Chatterton Yogesh Chaudhari Gautam Chaudhuri David Chaum diff --git a/Misc/NEWS.d/next/Library/2024-05-21-19-10-30.gh-issue-115225.eRmfJH.rst b/Misc/NEWS.d/next/Library/2024-05-21-19-10-30.gh-issue-115225.eRmfJH.rst new file mode 100644 index 00000000000000..2b65eaa6dd70ad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-21-19-10-30.gh-issue-115225.eRmfJH.rst @@ -0,0 +1 @@ +Raise error on certain technically valid but pathological ISO 8601 strings passed to :meth:`datetime.time.fromisoformat` that were previously parsed incorrectly. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index d6fa273c75e15e..bea6e9411a75ed 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1020,6 +1020,9 @@ parse_hh_mm_ss_ff(const char *tstr, const char *tstr_end, int *hour, continue; } else if (c == '.' || c == ',') { + if (i < 2) { + return -3; // Decimal mark on hour or minute + } break; } else if (!has_separator) { --p; From e83ce850f433fd8bbf8ff4e8d7649b942639db31 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 5 Jun 2024 18:54:50 +0100 Subject: [PATCH 390/903] pathlib ABCs: remove duplicate `realpath()` implementation. (#119178) Add private `posixpath._realpath()` function, which is a generic version of `realpath()` that can be parameterised with string tokens (`sep`, `curdir`, `pardir`) and query functions (`getcwd`, `lstat`, `readlink`). Also add support for limiting the number of symlink traversals. In the private `pathlib._abc.PathBase` class, call `posixpath._realpath()` and remove our re-implementation of the same algorithm. No change to any public APIs, either in `posixpath` or `pathlib`. Co-authored-by: Nice Zombies --- Lib/pathlib/_abc.py | 87 +++++++++++++++------------------------------ Lib/posixpath.py | 40 +++++++++++++++------ 2 files changed, 57 insertions(+), 70 deletions(-) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index d7471b6927331d..1a74f457c3f5a7 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -12,8 +12,8 @@ """ import functools +import posixpath from glob import _Globber, _no_recurse_symlinks -from errno import ENOTDIR, ELOOP from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO @@ -696,65 +696,34 @@ def resolve(self, strict=False): """ if self._resolving: return self - path_root, parts = self._stack - path = self.with_segments(path_root) - try: - path = path.absolute() - except UnsupportedOperation: - path_tail = [] - else: - path_root, path_tail = path._stack - path_tail.reverse() - - # If the user has *not* overridden the `readlink()` method, then symlinks are unsupported - # and (in non-strict mode) we can improve performance by not calling `stat()`. - querying = strict or getattr(self.readlink, '_supported', True) - link_count = 0 - while parts: - part = parts.pop() - if not part or part == '.': - continue - if part == '..': - if not path_tail: - if path_root: - # Delete '..' segment immediately following root - continue - elif path_tail[-1] != '..': - # Delete '..' segment and its predecessor - path_tail.pop() - continue - path_tail.append(part) - if querying and part != '..': - path = self.with_segments(path_root + self.parser.sep.join(path_tail)) + + def getcwd(): + return str(self.with_segments().absolute()) + + if strict or getattr(self.readlink, '_supported', True): + def lstat(path_str): + path = self.with_segments(path_str) path._resolving = True - try: - st = path.stat(follow_symlinks=False) - if S_ISLNK(st.st_mode): - # Like Linux and macOS, raise OSError(errno.ELOOP) if too many symlinks are - # encountered during resolution. - link_count += 1 - if link_count >= self._max_symlinks: - raise OSError(ELOOP, "Too many symbolic links in path", self._raw_path) - target_root, target_parts = path.readlink()._stack - # If the symlink target is absolute (like '/etc/hosts'), set the current - # path to its uppermost parent (like '/'). - if target_root: - path_root = target_root - path_tail.clear() - else: - path_tail.pop() - # Add the symlink target's reversed tail parts (like ['hosts', 'etc']) to - # the stack of unresolved path parts. - parts.extend(target_parts) - continue - elif parts and not S_ISDIR(st.st_mode): - raise NotADirectoryError(ENOTDIR, "Not a directory", self._raw_path) - except OSError: - if strict: - raise - else: - querying = False - return self.with_segments(path_root + self.parser.sep.join(path_tail)) + return path.lstat() + + def readlink(path_str): + path = self.with_segments(path_str) + path._resolving = True + return str(path.readlink()) + else: + # If the user has *not* overridden the `readlink()` method, then + # symlinks are unsupported and (in non-strict mode) we can improve + # performance by not calling `path.lstat()`. + def skip(path_str): + # This exception will be internally consumed by `_realpath()`. + raise OSError("Operation skipped.") + + lstat = readlink = skip + + return self.with_segments(posixpath._realpath( + str(self), strict, self.parser.sep, + getcwd=getcwd, lstat=lstat, readlink=readlink, + maxlinks=self._max_symlinks)) def symlink_to(self, target, target_is_directory=False): """ diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 47b2aa572e5c65..fccca4e066b76f 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -22,6 +22,7 @@ altsep = None devnull = '/dev/null' +import errno import os import sys import stat @@ -401,7 +402,10 @@ def realpath(filename, *, strict=False): curdir = '.' pardir = '..' getcwd = os.getcwd + return _realpath(filename, strict, sep, curdir, pardir, getcwd) +def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, + getcwd=os.getcwd, lstat=os.lstat, readlink=os.readlink, maxlinks=None): # The stack of unresolved path parts. When popped, a special value of None # indicates that a symlink target has been resolved, and that the original # symlink path can be retrieved by popping again. The [::-1] slice is a @@ -418,6 +422,10 @@ def realpath(filename, *, strict=False): # the same links. seen = {} + # Number of symlinks traversed. When the number of traversals is limited + # by *maxlinks*, this is used instead of *seen* to detect symlink loops. + link_count = 0 + while rest: name = rest.pop() if name is None: @@ -436,11 +444,19 @@ def realpath(filename, *, strict=False): else: newpath = path + sep + name try: - st = os.lstat(newpath) + st = lstat(newpath) if not stat.S_ISLNK(st.st_mode): path = newpath continue - if newpath in seen: + elif maxlinks is not None: + link_count += 1 + if link_count > maxlinks: + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + elif newpath in seen: # Already seen this path path = seen[newpath] if path is not None: @@ -448,26 +464,28 @@ def realpath(filename, *, strict=False): continue # The symlink is not resolved, so we must have a symlink loop. if strict: - # Raise OSError(errno.ELOOP) - os.stat(newpath) + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) path = newpath continue - target = os.readlink(newpath) + target = readlink(newpath) except OSError: if strict: raise path = newpath continue # Resolve the symbolic link - seen[newpath] = None # not resolved symlink if target.startswith(sep): # Symlink target is absolute; reset resolved path. path = sep - # Push the symlink path onto the stack, and signal its specialness by - # also pushing None. When these entries are popped, we'll record the - # fully-resolved symlink target in the 'seen' mapping. - rest.append(newpath) - rest.append(None) + if maxlinks is None: + # Mark this symlink as seen but not fully resolved. + seen[newpath] = None + # Push the symlink path onto the stack, and signal its specialness + # by also pushing None. When these entries are popped, we'll + # record the fully-resolved symlink target in the 'seen' mapping. + rest.append(newpath) + rest.append(None) # Push the unresolved symlink target parts onto the stack. rest.extend(target.split(sep)[::-1]) From f878d46e5614f08a9302fcb6fc611ef49e9acf2f Mon Sep 17 00:00:00 2001 From: Jan Kaliszewski Date: Wed, 5 Jun 2024 23:52:40 +0200 Subject: [PATCH 391/903] gh-120128: fix description of argument to ipaddress.collapse_addresses() (#120131) The argument to collapse_addresses() is now described as an *iterable* (rather than *iterator*). --- Doc/library/ipaddress.rst | 2 +- Lib/ipaddress.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index ead841b0581e21..f58c0ea75a4753 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -990,7 +990,7 @@ The module also provides the following module level functions: .. function:: collapse_addresses(addresses) Return an iterator of the collapsed :class:`IPv4Network` or - :class:`IPv6Network` objects. *addresses* is an iterator of + :class:`IPv6Network` objects. *addresses* is an :term:`iterable` of :class:`IPv4Network` or :class:`IPv6Network` objects. A :exc:`TypeError` is raised if *addresses* contains mixed version objects. diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 8e4d49c859534d..9cef275f7ae2fc 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -310,7 +310,7 @@ def collapse_addresses(addresses): [IPv4Network('192.0.2.0/24')] Args: - addresses: An iterator of IPv4Network or IPv6Network objects. + addresses: An iterable of IPv4Network or IPv6Network objects. Returns: An iterator of the collapsed IPv(4|6)Network objects. From eeb8f67f837facb37f092a8b743f4d249515e82f Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:56:58 +0100 Subject: [PATCH 392/903] gh-119786: move adaptive interpreter doc from Python to InternalsDoc (#120137) --- InternalDocs/README.md | 1 + {Python => InternalDocs}/adaptive.md | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) rename {Python => InternalDocs}/adaptive.md (93%) diff --git a/InternalDocs/README.md b/InternalDocs/README.md index e69e27d1542990..a2502fbf198735 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -14,3 +14,4 @@ it is not, please report that through the [Exception Handling](exception_handling.md) +[Adaptive Instruction Families](adaptive.md) diff --git a/Python/adaptive.md b/InternalDocs/adaptive.md similarity index 93% rename from Python/adaptive.md rename to InternalDocs/adaptive.md index d978c089b237e0..09245730b271fa 100644 --- a/Python/adaptive.md +++ b/InternalDocs/adaptive.md @@ -2,8 +2,9 @@ ## Families of instructions -The core part of PEP 659 (specializing adaptive interpreter) is the families -of instructions that perform the adaptive specialization. +The core part of [PEP 659](https://peps.python.org/pep-0659/) +(specializing adaptive interpreter) is the families of +instructions that perform the adaptive specialization. A family of instructions has the following fundamental properties: @@ -30,8 +31,9 @@ although these are not fundamental and may change: ## Example family -The `LOAD_GLOBAL` instruction (in Python/bytecodes.c) already has an adaptive -family that serves as a relatively simple example. +The `LOAD_GLOBAL` instruction (in +[Python/bytecodes.c](https://github.com/python/cpython/blob/main/Python/bytecodes.c)) +already has an adaptive family that serves as a relatively simple example. The `LOAD_GLOBAL` instruction performs adaptive specialization, calling `_Py_Specialize_LoadGlobal()` when the counter reaches zero. From fd104dfcb838d735ef8128e3539d7a730d403422 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Thu, 6 Jun 2024 13:40:37 +0200 Subject: [PATCH 393/903] gh-120111: Don't use cirrus M1 macOS runners on fork (#120116) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build.yml | 10 ++++++---- .github/workflows/reusable-macos.yml | 9 ++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cde93c77a0b82e..eb325ac2f9ee1b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -199,8 +199,9 @@ jobs: uses: ./.github/workflows/reusable-macos.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - # Cirrus is M1, macos-13 is default GHA Intel - os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-13"]' + # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. + # Cirrus used for upstream, macos-14 for forks. + os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-14", "macos-13"]' build_macos_free_threading: name: 'macOS (free-threading)' @@ -210,8 +211,9 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} free-threading: true - # Cirrus is M1 - os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma"]' + # Cirrus and macos-14 are M1. + # Cirrus used for upstream, macos-14 for forks. + os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-14"]' build_ubuntu: name: 'Ubuntu' diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index d06a718d199c96..f825d1a7b3f69a 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -14,7 +14,7 @@ on: jobs: build_macos: - name: 'build and test' + name: build and test (${{ matrix.os }}) timeout-minutes: 60 env: HOMEBREW_NO_ANALYTICS: 1 @@ -27,6 +27,13 @@ jobs: fail-fast: false matrix: os: ${{fromJson(inputs.os-matrix)}} + is-fork: + - ${{ github.repository_owner != 'python' }} + exclude: + - os: "ghcr.io/cirruslabs/macos-runner:sonoma" + is-fork: true + - os: "macos-14" + is-fork: false runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From cccc9f63c63ae693ccd0e2d8fc6cfd3aa18feb8e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 6 Jun 2024 16:11:42 +0100 Subject: [PATCH 394/903] gh-119679: Fix layout of PYD and DLL files on Windows when using PC/layout script (GH-120133) --- PC/layout/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PC/layout/main.py b/PC/layout/main.py index 716f01097fe3b0..0350ed7af3f9b5 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -202,7 +202,7 @@ def in_build(f, dest="", new_name=None, no_lib=False): yield "LICENSE.txt", ns.build / "LICENSE.txt" - dest="" if ns.flat_dlls else "DLLs/" + dest = "" if ns.flat_dlls else "DLLs/" for _, src in rglob(ns.build, "*.pyd"): if ns.include_freethreaded: @@ -226,7 +226,7 @@ def in_build(f, dest="", new_name=None, no_lib=False): continue if src in EXCLUDE_FROM_DLLS: continue - yield from in_build(src.name, no_lib=True) + yield from in_build(src.name, dest=dest, no_lib=True) if ns.zip_lib: zip_name = PYTHON_ZIP_NAME From 78634cfa3dd4b542897835d5f097604dbeb0f3fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 6 Jun 2024 17:31:33 +0200 Subject: [PATCH 395/903] gh-120155: Initialize variables in _tkinter.c (#120156) Initialize variables in _tkinter.c to make static analyzers happy. --- Modules/_tkinter.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 24f87c8d34c6b2..a34646aecb3ec8 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -1438,7 +1438,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) marshal the parameters to the interpreter thread. */ Tkapp_CallEvent *ev; Tcl_Condition cond = NULL; - PyObject *exc; + PyObject *exc = NULL; // init to make static analyzers happy if (!WaitForMainloop(self)) return NULL; ev = (Tkapp_CallEvent*)attemptckalloc(sizeof(Tkapp_CallEvent)); @@ -1712,7 +1712,8 @@ var_invoke(EventFunc func, PyObject *selfptr, PyObject *args, int flags) TkappObject *self = (TkappObject*)selfptr; if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { VarEvent *ev; - PyObject *res, *exc; + // init 'res' and 'exc' to make static analyzers happy + PyObject *res = NULL, *exc = NULL; Tcl_Condition cond = NULL; /* The current thread is not the interpreter thread. Marshal @@ -2413,6 +2414,8 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, data->self = self; data->func = Py_NewRef(func); if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { + err = 0; // init to make static analyzers happy + Tcl_Condition cond = NULL; CommandEvent *ev = (CommandEvent*)attemptckalloc(sizeof(CommandEvent)); if (ev == NULL) { @@ -2468,6 +2471,8 @@ _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name) TRACE(self, ("((sss))", "rename", name, "")); if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { + err = 0; // init to make static analyzers happy + Tcl_Condition cond = NULL; CommandEvent *ev; ev = (CommandEvent*)attemptckalloc(sizeof(CommandEvent)); From d50a7c478feb4037e65fcaea453d9ecc00259dd9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 6 Jun 2024 09:25:05 -0700 Subject: [PATCH 396/903] CODEOWNERS: Add myself to symtable and AST (#120139) Co-authored-by: Carl Meyer --- .github/CODEOWNERS | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ca5c8aaa3a0bef..c7021b3c2a4c40 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -40,6 +40,7 @@ Python/bytecodes.c @markshannon Python/optimizer*.c @markshannon Python/optimizer_analysis.c @Fidget-Spinner Python/optimizer_bytecodes.c @Fidget-Spinner +Python/symtable.c @JelleZijlstra @carljm Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv Lib/test/test_patma.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra @@ -153,10 +154,10 @@ Include/internal/pycore_time.h @pganssle @abalkin /Tools/cases_generator/ @markshannon # AST -Python/ast.c @isidentical -Parser/asdl.py @isidentical -Parser/asdl_c.py @isidentical -Lib/ast.py @isidentical +Python/ast.c @isidentical @JelleZijlstra +Parser/asdl.py @isidentical @JelleZijlstra +Parser/asdl_c.py @isidentical @JelleZijlstra +Lib/ast.py @isidentical @JelleZijlstra # Mock /Lib/unittest/mock.py @cjw296 From 2d7ff6e0e7d4c08ba84079a5c19a4a485626e1de Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 6 Jun 2024 20:12:32 +0300 Subject: [PATCH 397/903] Restore decimal context after decimal doctests (GH-120149) The modified context caused tests failures in several other tests. --- Lib/test/test_decimal.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index e927e24b582a5d..46755107de0102 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -5892,13 +5892,17 @@ def load_tests(loader, tests, pattern): if TODO_TESTS is None: from doctest import DocTestSuite, IGNORE_EXCEPTION_DETAIL + orig_context = orig_sys_decimal.getcontext().copy() for mod in C, P: if not mod: continue def setUp(slf, mod=mod): sys.modules['decimal'] = mod - def tearDown(slf): + init(mod) + def tearDown(slf, mod=mod): sys.modules['decimal'] = orig_sys_decimal + mod.setcontext(ORIGINAL_CONTEXT[mod].copy()) + orig_sys_decimal.setcontext(orig_context.copy()) optionflags = IGNORE_EXCEPTION_DETAIL if mod is C else 0 sys.modules['decimal'] = mod tests.addTest(DocTestSuite(mod, setUp=setUp, tearDown=tearDown, @@ -5913,8 +5917,8 @@ def setUpModule(): TEST_ALL = ARITH if ARITH is not None else is_resource_enabled('decimal') def tearDownModule(): - if C: C.setcontext(ORIGINAL_CONTEXT[C]) - P.setcontext(ORIGINAL_CONTEXT[P]) + if C: C.setcontext(ORIGINAL_CONTEXT[C].copy()) + P.setcontext(ORIGINAL_CONTEXT[P].copy()) if not C: warnings.warn('C tests skipped: no module named _decimal.', UserWarning) From 417bec733c11e63df559ecf898802dbef590142e Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 6 Jun 2024 10:20:37 -0700 Subject: [PATCH 398/903] Add Tian Gao to CODEOWNERS and ACKS (GH-120166) --- .github/CODEOWNERS | 4 ++++ Misc/ACKS | 1 + 2 files changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c7021b3c2a4c40..811b8cfdab17dc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -174,6 +174,10 @@ Lib/ast.py @isidentical @JelleZijlstra /Lib/test/test_subprocess.py @gpshead /Modules/*subprocess* @gpshead +# debugger +**/*pdb* @gaogaotiantian +**/*bdb* @gaogaotiantian + # Limited C API & stable ABI Tools/build/stable_abi.py @encukou Misc/stable_abi.toml @encukou diff --git a/Misc/ACKS b/Misc/ACKS index af92d81ff3141a..2f4c0793437fb6 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -610,6 +610,7 @@ Nitin Ganatra Soumendra Ganguly (गङ्गोपाध्याय) Fred Gansevles Paul Ganssle +Tian Gao Lars Marius Garshol Jake Garver Dan Gass From e21057b99967eb5323320e6d1121955e0cd2985e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 6 Jun 2024 13:40:58 -0400 Subject: [PATCH 399/903] gh-117657: Fix TSAN race involving import lock (#118523) This adds a `_PyRecursiveMutex` type based on `PyMutex` and uses that for the import lock. This fixes some data races in the free-threaded build and generally simplifies the import lock code. --- Include/internal/pycore_import.h | 18 +---- Include/internal/pycore_lock.h | 12 ++++ Modules/_testinternalcapi/test_lock.c | 25 +++++++ Modules/posixmodule.c | 11 +-- Python/import.c | 83 ++-------------------- Python/lock.c | 42 +++++++++++ Tools/tsan/suppressions_free_threading.txt | 4 -- 7 files changed, 90 insertions(+), 105 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index f8329a460d6cbf..290ba95e1a0ad7 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -20,7 +20,7 @@ PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); extern int _PyImport_SetModuleString(const char *name, PyObject* module); extern void _PyImport_AcquireLock(PyInterpreterState *interp); -extern int _PyImport_ReleaseLock(PyInterpreterState *interp); +extern void _PyImport_ReleaseLock(PyInterpreterState *interp); // This is used exclusively for the sys and builtins modules: extern int _PyImport_FixupBuiltin( @@ -94,11 +94,7 @@ struct _import_state { #endif PyObject *import_func; /* The global import lock. */ - struct { - PyThread_type_lock mutex; - unsigned long thread; - int level; - } lock; + _PyRecursiveMutex lock; /* diagnostic info in PyImport_ImportModuleLevelObject() */ struct { int import_level; @@ -123,11 +119,6 @@ struct _import_state { #define IMPORTS_INIT \ { \ DLOPENFLAGS_INIT \ - .lock = { \ - .mutex = NULL, \ - .thread = PYTHREAD_INVALID_THREAD_ID, \ - .level = 0, \ - }, \ .find_and_load = { \ .header = 1, \ }, \ @@ -180,11 +171,6 @@ extern void _PyImport_FiniCore(PyInterpreterState *interp); extern void _PyImport_FiniExternal(PyInterpreterState *interp); -#ifdef HAVE_FORK -extern PyStatus _PyImport_ReInitLock(PyInterpreterState *interp); -#endif - - extern PyObject* _PyImport_GetBuiltinModuleNames(void); struct _module_alias { diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index a5b28e4bd4744e..d5853b2c9ff464 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -219,6 +219,18 @@ _PyOnceFlag_CallOnce(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) return _PyOnceFlag_CallOnceSlow(flag, fn, arg); } +// A recursive mutex. The mutex should zero-initialized. +typedef struct { + PyMutex mutex; + unsigned long long thread; // i.e., PyThread_get_thread_ident_ex() + size_t level; +} _PyRecursiveMutex; + +PyAPI_FUNC(int) _PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m); +PyAPI_FUNC(void) _PyRecursiveMutex_Lock(_PyRecursiveMutex *m); +PyAPI_FUNC(void) _PyRecursiveMutex_Unlock(_PyRecursiveMutex *m); + + // A readers-writer (RW) lock. The lock supports multiple concurrent readers or // a single writer. The lock is write-preferring: if a writer is waiting while // the lock is read-locked then, new readers will be blocked. This avoids diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 4900459c689279..1544fe1363c7c5 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -2,6 +2,7 @@ #include "parts.h" #include "pycore_lock.h" +#include "pycore_pythread.h" // PyThread_get_thread_ident_ex() #include "clinic/test_lock.c.h" @@ -476,6 +477,29 @@ test_lock_rwlock(PyObject *self, PyObject *obj) Py_RETURN_NONE; } +static PyObject * +test_lock_recursive(PyObject *self, PyObject *obj) +{ + _PyRecursiveMutex m = (_PyRecursiveMutex){0}; + assert(!_PyRecursiveMutex_IsLockedByCurrentThread(&m)); + + _PyRecursiveMutex_Lock(&m); + assert(m.thread == PyThread_get_thread_ident_ex()); + assert(PyMutex_IsLocked(&m.mutex)); + assert(m.level == 0); + + _PyRecursiveMutex_Lock(&m); + assert(m.level == 1); + _PyRecursiveMutex_Unlock(&m); + + _PyRecursiveMutex_Unlock(&m); + assert(m.thread == 0); + assert(!PyMutex_IsLocked(&m.mutex)); + assert(m.level == 0); + + Py_RETURN_NONE; +} + static PyMethodDef test_methods[] = { {"test_lock_basic", test_lock_basic, METH_NOARGS}, {"test_lock_two_threads", test_lock_two_threads, METH_NOARGS}, @@ -485,6 +509,7 @@ static PyMethodDef test_methods[] = { {"test_lock_benchmark", test_lock_benchmark, METH_NOARGS}, {"test_lock_once", test_lock_once, METH_NOARGS}, {"test_lock_rwlock", test_lock_rwlock, METH_NOARGS}, + {"test_lock_recursive", test_lock_recursive, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 386e942d53f539..5f943d4b1c8085 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16,7 +16,6 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_ReInitThreads() #include "pycore_fileutils.h" // _Py_closerange() -#include "pycore_import.h" // _PyImport_ReInitLock() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_moduleobject.h" // _PyModule_GetState() @@ -627,10 +626,7 @@ PyOS_AfterFork_Parent(void) _PyEval_StartTheWorldAll(&_PyRuntime); PyInterpreterState *interp = _PyInterpreterState_GET(); - if (_PyImport_ReleaseLock(interp) <= 0) { - Py_FatalError("failed releasing import lock after fork"); - } - + _PyImport_ReleaseLock(interp); run_at_forkers(interp->after_forkers_parent, 0); } @@ -675,10 +671,7 @@ PyOS_AfterFork_Child(void) _PyEval_StartTheWorldAll(&_PyRuntime); _PyThreadState_DeleteList(list); - status = _PyImport_ReInitLock(tstate->interp); - if (_PyStatus_EXCEPTION(status)) { - goto fatal_error; - } + _PyImport_ReleaseLock(tstate->interp); _PySignal_AfterFork(); diff --git a/Python/import.c b/Python/import.c index 351d463dcab465..2c7a461ac786c8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -94,11 +94,7 @@ static struct _inittab *inittab_copy = NULL; (interp)->imports.import_func #define IMPORT_LOCK(interp) \ - (interp)->imports.lock.mutex -#define IMPORT_LOCK_THREAD(interp) \ - (interp)->imports.lock.thread -#define IMPORT_LOCK_LEVEL(interp) \ - (interp)->imports.lock.level + (interp)->imports.lock #define FIND_AND_LOAD(interp) \ (interp)->imports.find_and_load @@ -115,74 +111,14 @@ static struct _inittab *inittab_copy = NULL; void _PyImport_AcquireLock(PyInterpreterState *interp) { - unsigned long me = PyThread_get_thread_ident(); - if (me == PYTHREAD_INVALID_THREAD_ID) - return; /* Too bad */ - if (IMPORT_LOCK(interp) == NULL) { - IMPORT_LOCK(interp) = PyThread_allocate_lock(); - if (IMPORT_LOCK(interp) == NULL) - return; /* Nothing much we can do. */ - } - if (IMPORT_LOCK_THREAD(interp) == me) { - IMPORT_LOCK_LEVEL(interp)++; - return; - } - if (IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID || - !PyThread_acquire_lock(IMPORT_LOCK(interp), 0)) - { - PyThreadState *tstate = PyEval_SaveThread(); - PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); - PyEval_RestoreThread(tstate); - } - assert(IMPORT_LOCK_LEVEL(interp) == 0); - IMPORT_LOCK_THREAD(interp) = me; - IMPORT_LOCK_LEVEL(interp) = 1; + _PyRecursiveMutex_Lock(&IMPORT_LOCK(interp)); } -int +void _PyImport_ReleaseLock(PyInterpreterState *interp) { - unsigned long me = PyThread_get_thread_ident(); - if (me == PYTHREAD_INVALID_THREAD_ID || IMPORT_LOCK(interp) == NULL) - return 0; /* Too bad */ - if (IMPORT_LOCK_THREAD(interp) != me) - return -1; - IMPORT_LOCK_LEVEL(interp)--; - assert(IMPORT_LOCK_LEVEL(interp) >= 0); - if (IMPORT_LOCK_LEVEL(interp) == 0) { - IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; - PyThread_release_lock(IMPORT_LOCK(interp)); - } - return 1; -} - -#ifdef HAVE_FORK -/* This function is called from PyOS_AfterFork_Child() to ensure that newly - created child processes do not share locks with the parent. - We now acquire the import lock around fork() calls but on some platforms - (Solaris 9 and earlier? see isue7242) that still left us with problems. */ -PyStatus -_PyImport_ReInitLock(PyInterpreterState *interp) -{ - if (IMPORT_LOCK(interp) != NULL) { - if (_PyThread_at_fork_reinit(&IMPORT_LOCK(interp)) < 0) { - return _PyStatus_ERR("failed to create a new lock"); - } - } - - if (IMPORT_LOCK_LEVEL(interp) > 1) { - /* Forked as a side effect of import */ - unsigned long me = PyThread_get_thread_ident(); - PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); - IMPORT_LOCK_THREAD(interp) = me; - IMPORT_LOCK_LEVEL(interp)--; - } else { - IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; - IMPORT_LOCK_LEVEL(interp) = 0; - } - return _PyStatus_OK(); + _PyRecursiveMutex_Unlock(&IMPORT_LOCK(interp)); } -#endif /***************/ @@ -4111,11 +4047,6 @@ _PyImport_FiniCore(PyInterpreterState *interp) PyErr_FormatUnraisable("Exception ignored on clearing sys.modules"); } - if (IMPORT_LOCK(interp) != NULL) { - PyThread_free_lock(IMPORT_LOCK(interp)); - IMPORT_LOCK(interp) = NULL; - } - _PyImport_ClearCore(interp); } @@ -4248,8 +4179,7 @@ _imp_lock_held_impl(PyObject *module) /*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyBool_FromLong( - IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID); + return PyBool_FromLong(PyMutex_IsLocked(&IMPORT_LOCK(interp).mutex)); } /*[clinic input] @@ -4283,11 +4213,12 @@ _imp_release_lock_impl(PyObject *module) /*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (_PyImport_ReleaseLock(interp) < 0) { + if (!_PyRecursiveMutex_IsLockedByCurrentThread(&IMPORT_LOCK(interp))) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; } + _PyImport_ReleaseLock(interp); Py_RETURN_NONE; } diff --git a/Python/lock.c b/Python/lock.c index 239e56ad929ea3..555f4c25b9b214 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -366,6 +366,48 @@ _PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) } } +static int +recursive_mutex_is_owned_by(_PyRecursiveMutex *m, PyThread_ident_t tid) +{ + return _Py_atomic_load_ullong_relaxed(&m->thread) == tid; +} + +int +_PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m) +{ + return recursive_mutex_is_owned_by(m, PyThread_get_thread_ident_ex()); +} + +void +_PyRecursiveMutex_Lock(_PyRecursiveMutex *m) +{ + PyThread_ident_t thread = PyThread_get_thread_ident_ex(); + if (recursive_mutex_is_owned_by(m, thread)) { + m->level++; + return; + } + PyMutex_Lock(&m->mutex); + _Py_atomic_store_ullong_relaxed(&m->thread, thread); + assert(m->level == 0); +} + +void +_PyRecursiveMutex_Unlock(_PyRecursiveMutex *m) +{ + PyThread_ident_t thread = PyThread_get_thread_ident_ex(); + if (!recursive_mutex_is_owned_by(m, thread)) { + Py_FatalError("unlocking a recursive mutex that is not owned by the" + " current thread"); + } + if (m->level > 0) { + m->level--; + return; + } + assert(m->level == 0); + _Py_atomic_store_ullong_relaxed(&m->thread, 0); + PyMutex_Unlock(&m->mutex); +} + #define _Py_WRITE_LOCKED 1 #define _PyRWMutex_READER_SHIFT 2 #define _Py_RWMUTEX_MAX_READERS (UINTPTR_MAX >> _PyRWMutex_READER_SHIFT) diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 8b64d1ff321858..cb48a30751ac7b 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -26,8 +26,6 @@ race:free_threadstate race_top:_add_to_weak_set race_top:_in_weak_set race_top:_PyEval_EvalFrameDefault -race_top:_PyImport_AcquireLock -race_top:_PyImport_ReleaseLock race_top:_PyType_HasFeature race_top:assign_version_tag race_top:insertdict @@ -41,9 +39,7 @@ race_top:set_discard_entry race_top:set_inheritable race_top:Py_SET_TYPE race_top:_PyDict_CheckConsistency -race_top:_PyImport_AcquireLock race_top:_Py_dict_lookup_threadsafe -race_top:_imp_release_lock race_top:_multiprocessing_SemLock_acquire_impl race_top:dictiter_new race_top:dictresize From 5bdc87b8859c837092e7c5b19583f98488f7a387 Mon Sep 17 00:00:00 2001 From: David Lowry-Duda Date: Thu, 6 Jun 2024 17:35:24 -0400 Subject: [PATCH 400/903] gh-120178: Documentation typo corrections (#120179) --- Doc/glossary.rst | 2 +- Doc/library/dis.rst | 2 +- Doc/library/pdb.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index ae9949bc2867c4..8685369117fd87 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -594,7 +594,7 @@ Glossary therefore it is never deallocated. Built-in strings and singletons are immortal objects. For example, - :const:`True` and :const:`None` singletons are immmortal. + :const:`True` and :const:`None` singletons are immortal. See `PEP 683 – Immortal Objects, Using a Fixed Refcount `_ for more information. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 87d1bcdfaf3f1d..ab46d4554d8773 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1667,7 +1667,7 @@ iterations of the loop. A no-op. Performs internal tracing, debugging and optimization checks. - The ``context`` oparand consists of two parts. The lowest two bits + The ``context`` operand consists of two parts. The lowest two bits indicate where the ``RESUME`` occurs: * ``0`` The start of a function, which is neither a generator, coroutine diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index cd6496203949ea..f6085171dccb38 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -341,7 +341,7 @@ can be overridden by the local file. With a *lineno* argument, set a break at line *lineno* in the current file. The line number may be prefixed with a *filename* and a colon, to specify a breakpoint in another file (possibly one that hasn't been loaded - yet). The file is searched on :data:`sys.path`. Accepatable forms of *filename* + yet). The file is searched on :data:`sys.path`. Acceptable forms of *filename* are ``/abspath/to/file.py``, ``relpath/file.py``, ``module`` and ``package.module``. From 14e1506a6d7056c38fbbc0797268dcf783f91243 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 7 Jun 2024 00:27:39 +0100 Subject: [PATCH 401/903] GH-119054: Add "Reading directories" section to pathlib docs (#119956) Add a dedicated subsection for `Path.iterdir()`-related methods, specifically `iterdir()`, `glob()`, `rglob()` and `walk()`. Co-authored-by: Jelle Zijlstra --- Doc/library/pathlib.rst | 197 +++++++++++++++++++++------------------- 1 file changed, 102 insertions(+), 95 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index f37bb33321fa53..b7ab44706a0160 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -820,6 +820,9 @@ bugs or failures in your application):: % (cls.__name__,)) UnsupportedOperation: cannot instantiate 'WindowsPath' on your system +Some concrete path methods can raise an :exc:`OSError` if a system call fails +(for example because the path doesn't exist). + Parsing and generating URIs ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1151,69 +1154,32 @@ Reading and writing files .. versionadded:: 3.5 -Other methods -^^^^^^^^^^^^^ - -Some of these methods can raise an :exc:`OSError` if a system call fails (for -example because the path doesn't exist). - - -.. classmethod:: Path.cwd() - - Return a new path object representing the current directory (as returned - by :func:`os.getcwd`):: - - >>> Path.cwd() - PosixPath('/home/antoine/pathlib') - - -.. classmethod:: Path.home() - - Return a new path object representing the user's home directory (as - returned by :func:`os.path.expanduser` with ``~`` construct). If the home - directory can't be resolved, :exc:`RuntimeError` is raised. - - :: - - >>> Path.home() - PosixPath('/home/antoine') - - .. versionadded:: 3.5 - - -.. method:: Path.chmod(mode, *, follow_symlinks=True) - - Change the file mode and permissions, like :func:`os.chmod`. - - This method normally follows symlinks. Some Unix flavours support changing - permissions on the symlink itself; on these platforms you may add the - argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. - - :: - - >>> p = Path('setup.py') - >>> p.stat().st_mode - 33277 - >>> p.chmod(0o444) - >>> p.stat().st_mode - 33060 - - .. versionchanged:: 3.10 - The *follow_symlinks* parameter was added. +Reading directories +^^^^^^^^^^^^^^^^^^^ -.. method:: Path.expanduser() +.. method:: Path.iterdir() - Return a new path with expanded ``~`` and ``~user`` constructs, - as returned by :meth:`os.path.expanduser`. If a home directory can't be - resolved, :exc:`RuntimeError` is raised. + When the path points to a directory, yield path objects of the directory + contents:: - :: + >>> p = Path('docs') + >>> for child in p.iterdir(): child + ... + PosixPath('docs/conf.py') + PosixPath('docs/_templates') + PosixPath('docs/make.bat') + PosixPath('docs/index.rst') + PosixPath('docs/_build') + PosixPath('docs/_static') + PosixPath('docs/Makefile') - >>> p = PosixPath('~/films/Monty Python') - >>> p.expanduser() - PosixPath('/home/eric/films/Monty Python') + The children are yielded in arbitrary order, and the special entries + ``'.'`` and ``'..'`` are not included. If a file is removed from or added + to the directory after creating the iterator, it is unspecified whether + a path object for that file is included. - .. versionadded:: 3.5 + If the path is not a directory or otherwise inaccessible, :exc:`OSError` is + raised. .. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False) @@ -1281,43 +1247,6 @@ example because the path doesn't exist). The *pattern* parameter accepts a :term:`path-like object`. -.. method:: Path.group(*, follow_symlinks=True) - - Return the name of the group owning the file. :exc:`KeyError` is raised - if the file's gid isn't found in the system database. - - This method normally follows symlinks; to get the group of the symlink, add - the argument ``follow_symlinks=False``. - - .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not - available. In previous versions, :exc:`NotImplementedError` was raised. - - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. - - -.. method:: Path.iterdir() - - When the path points to a directory, yield path objects of the directory - contents:: - - >>> p = Path('docs') - >>> for child in p.iterdir(): child - ... - PosixPath('docs/conf.py') - PosixPath('docs/_templates') - PosixPath('docs/make.bat') - PosixPath('docs/index.rst') - PosixPath('docs/_build') - PosixPath('docs/_static') - PosixPath('docs/Makefile') - - The children are yielded in arbitrary order, and the special entries - ``'.'`` and ``'..'`` are not included. If a file is removed from or added - to the directory after creating the iterator, whether a path object for - that file be included is unspecified. - .. method:: Path.walk(top_down=True, on_error=None, follow_symlinks=False) Generate the file names in a directory tree by walking the tree @@ -1413,6 +1342,84 @@ example because the path doesn't exist). .. versionadded:: 3.12 + +Other methods +^^^^^^^^^^^^^ + +.. classmethod:: Path.cwd() + + Return a new path object representing the current directory (as returned + by :func:`os.getcwd`):: + + >>> Path.cwd() + PosixPath('/home/antoine/pathlib') + + +.. classmethod:: Path.home() + + Return a new path object representing the user's home directory (as + returned by :func:`os.path.expanduser` with ``~`` construct). If the home + directory can't be resolved, :exc:`RuntimeError` is raised. + + :: + + >>> Path.home() + PosixPath('/home/antoine') + + .. versionadded:: 3.5 + + +.. method:: Path.chmod(mode, *, follow_symlinks=True) + + Change the file mode and permissions, like :func:`os.chmod`. + + This method normally follows symlinks. Some Unix flavours support changing + permissions on the symlink itself; on these platforms you may add the + argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. + + :: + + >>> p = Path('setup.py') + >>> p.stat().st_mode + 33277 + >>> p.chmod(0o444) + >>> p.stat().st_mode + 33060 + + .. versionchanged:: 3.10 + The *follow_symlinks* parameter was added. + +.. method:: Path.expanduser() + + Return a new path with expanded ``~`` and ``~user`` constructs, + as returned by :meth:`os.path.expanduser`. If a home directory can't be + resolved, :exc:`RuntimeError` is raised. + + :: + + >>> p = PosixPath('~/films/Monty Python') + >>> p.expanduser() + PosixPath('/home/eric/films/Monty Python') + + .. versionadded:: 3.5 + + +.. method:: Path.group(*, follow_symlinks=True) + + Return the name of the group owning the file. :exc:`KeyError` is raised + if the file's gid isn't found in the system database. + + This method normally follows symlinks; to get the group of the symlink, add + the argument ``follow_symlinks=False``. + + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not + available. In previous versions, :exc:`NotImplementedError` was raised. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + + .. method:: Path.lchmod(mode) Like :meth:`Path.chmod` but, if the path points to a symbolic link, the From 6b606522ca97488aad6fe2f193d4511e7a8f8334 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Thu, 6 Jun 2024 23:18:30 -0400 Subject: [PATCH 402/903] gh-119577: Adjust DeprecationWarning when testing element truth values in ElementTree (GH-119762) Adjust DeprecationWarning when testing element truth values in ElementTree, we're planning to go with the more natural True return rather than a disruptive harder to code around exception raise, and are deferring the behavior change for a few more releases. --- Doc/library/xml.etree.elementtree.rst | 7 ++++--- Doc/whatsnew/3.12.rst | 7 +++++-- Doc/whatsnew/3.13.rst | 10 +++++----- Lib/test/test_xml_etree.py | 2 +- Lib/xml/etree/ElementTree.py | 2 +- .../2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst | 4 ++++ Modules/_elementtree.c | 2 +- 7 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index a6a9eb87f56d88..e5919029c62c93 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1058,9 +1058,10 @@ Element Objects :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__len__`. - Caution: Elements with no subelements will test as ``False``. Testing the - truth value of an Element is deprecated and will raise an exception in - Python 3.14. Use specific ``len(elem)`` or ``elem is None`` test instead.:: + Caution: Elements with no subelements will test as ``False``. In a future + release of Python, all elements will test as ``True`` regardless of whether + subelements exist. Instead, prefer explicit ``len(elem)`` or + ``elem is not None`` tests.:: element = root.find('foo') diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index f99489fb53db74..28b28e9ce50e11 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1440,8 +1440,6 @@ and will be removed in Python 3.14. * :mod:`typing`: :class:`!typing.ByteString` -* :mod:`xml.etree.ElementTree`: Testing the truth value of an :class:`xml.etree.ElementTree.Element`. - * The ``__package__`` and ``__cached__`` attributes on module objects. * The :attr:`~codeobject.co_lnotab` attribute of code objects. @@ -1467,6 +1465,11 @@ although there is currently no date scheduled for their removal. * :class:`typing.Text` (:gh:`92332`) +* :mod:`xml.etree.ElementTree`: Testing the truth value of an + :class:`xml.etree.ElementTree.Element` is deprecated. In a future release it + will always return True. Prefer explicit ``len(elem)`` or + ``elem is not None`` tests instead. + * Currently Python accepts numeric literals immediately followed by keywords, for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing and ambiguous expressions like ``[0x1for x in y]`` (which can be diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a1d2a0d84e7581..81daaabdb889d7 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1728,11 +1728,6 @@ Pending Removal in Python 3.14 public API. (Contributed by Gregory P. Smith in :gh:`88168`.) -* :mod:`xml.etree.ElementTree`: Testing the truth value of an - :class:`~xml.etree.ElementTree.Element` is deprecated and will raise an - exception in Python 3.14. - - Pending Removal in Python 3.15 ------------------------------ @@ -1937,6 +1932,11 @@ although there is currently no date scheduled for their removal. * :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial writes. +* :mod:`xml.etree.ElementTree`: Testing the truth value of an + :class:`~xml.etree.ElementTree.Element` is deprecated. In a future release it + it will always return ``True``. Prefer explicit ``len(elem)`` or + ``elem is not None`` tests instead. + * :meth:`zipimport.zipimporter.load_module` is deprecated: use :meth:`~zipimport.zipimporter.exec_module` instead. diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index bae61f754e75f5..3d9141fea1ef3e 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -4088,7 +4088,7 @@ class BoolTest(unittest.TestCase): def test_warning(self): e = ET.fromstring('') msg = ( - r"Testing an element's truth value will raise an exception in " + r"Testing an element's truth value will always return True in " r"future versions. " r"Use specific 'len\(elem\)' or 'elem is not None' test instead.") with self.assertWarnsRegex(DeprecationWarning, msg): diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 9e15d34d22aa6c..ce67d7d7d54748 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -201,7 +201,7 @@ def __len__(self): def __bool__(self): warnings.warn( - "Testing an element's truth value will raise an exception in " + "Testing an element's truth value will always return True in " "future versions. " "Use specific 'len(elem)' or 'elem is not None' test instead.", DeprecationWarning, stacklevel=2 diff --git a/Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst b/Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst new file mode 100644 index 00000000000000..bd2daf3fb5c16d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-29-21-50-05.gh-issue-119577.S3BlKJ.rst @@ -0,0 +1,4 @@ +The :exc:`DeprecationWarning` emitted when testing the truth value of an +:class:`xml.etree.ElementTree.Element` now describes unconditionally +returning ``True`` in a future version rather than raising an exception in +Python 3.14. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index b11983d2caa2d1..3818e20b4f0f28 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1502,7 +1502,7 @@ element_bool(PyObject* self_) { ElementObject* self = (ElementObject*) self_; if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Testing an element's truth value will raise an exception " + "Testing an element's truth value will always return True " "in future versions. Use specific 'len(elem)' or " "'elem is not None' test instead.", 1) < 0) { From 5c115567b1e3aecb7a53cfd5757e25c088398411 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 7 Jun 2024 00:38:31 -0600 Subject: [PATCH 403/903] Add Plausible for docs metrics (#119977) Co-authored-by: Julien Palard --- Doc/conf.py | 3 ++- Doc/tools/templates/layout.html | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Doc/conf.py b/Doc/conf.py index 47fb96fe1de482..8a14646801ebac 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -339,7 +339,8 @@ html_context = { "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external", "repository_url": repository_url.removesuffix(".git") if repository_url else None, - "pr_id": os.getenv("READTHEDOCS_VERSION") + "pr_id": os.getenv("READTHEDOCS_VERSION"), + "enable_analytics": os.getenv("PYTHON_DOCS_ENABLE_ANALYTICS"), } # This 'Last updated on:' timestamp is inserted at the bottom of every page. diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 3f88fc8e91faad..b09fd21a8ddcc9 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,6 +26,9 @@ {% endblock %} {% block extrahead %} + {% if builder == "html" and enable_analytics %} + + {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} From bd826b9c77dbf7c789433cb8061c733c08634c0e Mon Sep 17 00:00:00 2001 From: Clinton Date: Fri, 7 Jun 2024 03:39:19 -0400 Subject: [PATCH 404/903] gh-120157: Remove unused code in concurrent.future (gh-120187) --- Lib/concurrent/futures/_base.py | 8 -------- .../2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst | 1 + 2 files changed, 1 insertion(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index 6742a07753c921..707fcdfde79acd 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -23,14 +23,6 @@ CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED' FINISHED = 'FINISHED' -_FUTURE_STATES = [ - PENDING, - RUNNING, - CANCELLED, - CANCELLED_AND_NOTIFIED, - FINISHED -] - _STATE_TO_DESCRIPTION_MAP = { PENDING: "pending", RUNNING: "running", diff --git a/Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst b/Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst new file mode 100644 index 00000000000000..3e905125797af7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-07-02-00-31.gh-issue-120157.HnWcF9.rst @@ -0,0 +1 @@ +Remove unused constant ``concurrent.futures._base._FUTURE_STATES`` in :mod:`concurrent.futures`. Patch by Clinton Christian (pygeek). From 57ad769076201c858a768d81047f6ea44925a33b Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Fri, 7 Jun 2024 11:03:28 +0300 Subject: [PATCH 405/903] gh-120080: Accept ``None`` as a valid argument for direct call of the ``int.__round__`` (#120088) Co-authored-by: Nikita Sobolev --- Lib/test/test_float.py | 6 ++++++ Lib/test/test_inspect/test_inspect.py | 1 - Lib/test/test_int.py | 6 ++++++ .../2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst | 2 ++ Objects/clinic/longobject.c.h | 6 +++--- Objects/longobject.c | 6 +++--- 6 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 5bd640617d6874..53695cefb8fded 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -949,6 +949,12 @@ def test_None_ndigits(self): self.assertEqual(x, 2) self.assertIsInstance(x, int) + def test_round_with_none_arg_direct_call(self): + for val in [(1.0).__round__(None), + round(1.0), + round(1.0, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) # Beginning with Python 2.6 float has cross platform compatible # ways to create and represent inf and nan diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 011d42f34b6461..65007c16203c6d 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -5412,7 +5412,6 @@ def test_builtins_have_signatures(self): 'bytearray': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, 'bytes': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, 'dict': {'pop'}, - 'int': {'__round__'}, 'memoryview': {'cast', 'hex'}, 'str': {'count', 'endswith', 'find', 'index', 'maketrans', 'rfind', 'rindex', 'startswith'}, } diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index ce9febd741bba2..77221dfb6d5aa2 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -517,6 +517,12 @@ def test_issue31619(self): self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + def test_round_with_none_arg_direct_call(self): + for val in [(1).__round__(None), + round(1), + round(1, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) class IntStrDigitLimitsTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst new file mode 100644 index 00000000000000..8c5602fcdb4ad2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-08-39-40.gh-issue-120080.DJFK11.rst @@ -0,0 +1,2 @@ +Direct call to the :meth:`!int.__round__` now accepts ``None`` +as a valid argument. diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index 56bc3864582dcb..90375b9a082cca 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -116,7 +116,7 @@ int___format__(PyObject *self, PyObject *arg) } PyDoc_STRVAR(int___round____doc__, -"__round__($self, ndigits=, /)\n" +"__round__($self, ndigits=None, /)\n" "--\n" "\n" "Rounding an Integral returns itself.\n" @@ -133,7 +133,7 @@ static PyObject * int___round__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *o_ndigits = NULL; + PyObject *o_ndigits = Py_None; if (!_PyArg_CheckPositional("__round__", nargs, 0, 1)) { goto exit; @@ -476,4 +476,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=2ba2d8dcda9b99da input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a53f5ba9a6c16737 input=a9049054013a1b77]*/ diff --git a/Objects/longobject.c b/Objects/longobject.c index ee0b2a038a2aab..a3a59a20f0bb97 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6045,7 +6045,7 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) /*[clinic input] int.__round__ - ndigits as o_ndigits: object = NULL + ndigits as o_ndigits: object = None / Rounding an Integral returns itself. @@ -6055,7 +6055,7 @@ Rounding with an ndigits argument also returns an integer. static PyObject * int___round___impl(PyObject *self, PyObject *o_ndigits) -/*[clinic end generated code: output=954fda6b18875998 input=1614cf23ec9e18c3]*/ +/*[clinic end generated code: output=954fda6b18875998 input=30c2aec788263144]*/ { PyObject *temp, *result, *ndigits; @@ -6073,7 +6073,7 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) * * m - divmod_near(m, 10**n)[1]. */ - if (o_ndigits == NULL) + if (o_ndigits == Py_None) return long_long(self); ndigits = _PyNumber_Index(o_ndigits); From 6a97929a5ad76c55bc6e1cf32898a1c31093334d Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Fri, 7 Jun 2024 16:19:41 +0800 Subject: [PATCH 406/903] Fix typos in comments (#120188) --- Include/ceval.h | 2 +- Include/cpython/object.h | 2 +- Include/cpython/pyframe.h | 2 +- Include/internal/mimalloc/mimalloc/atomic.h | 2 +- Include/internal/mimalloc/mimalloc/internal.h | 2 +- Include/internal/pycore_initconfig.h | 2 +- Include/internal/pycore_instruments.h | 2 +- Include/internal/pycore_lock.h | 2 +- Include/internal/pycore_pythread.h | 2 +- Include/internal/pycore_unicodeobject.h | 2 +- Include/structmember.h | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index 1ec746c3708220..e9df8684996e23 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -42,7 +42,7 @@ PyAPI_FUNC(int) Py_MakePendingCalls(void); level exceeds "current recursion limit + 50". By construction, this protection can only be triggered when the "overflowed" flag is set. It means the cleanup code has itself gone into an infinite loop, or the - RecursionError has been mistakingly ignored. When this protection is + RecursionError has been mistakenly ignored. When this protection is triggered, the interpreter aborts with a Fatal Error. In addition, the "overflowed" flag is automatically reset when the diff --git a/Include/cpython/object.h b/Include/cpython/object.h index e624326693d8e7..0ab94e5e2a15e5 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -313,7 +313,7 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); * triggered as a side-effect of `dst` getting torn down no longer believes * `dst` points to a valid object. * - * Temporary variables are used to only evalutate macro arguments once and so + * Temporary variables are used to only evaluate macro arguments once and so * avoid the duplication of side effects. _Py_TYPEOF() or memcpy() is used to * avoid a miscompilation caused by type punning. See Py_CLEAR() comment for * implementation details about type punning. diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index eeafbb17a56bad..51529763923ec3 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -28,7 +28,7 @@ struct _PyInterpreterFrame; * Does not raise an exception. */ PyAPI_FUNC(PyObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); -/* Returns a byte ofsset into the last executed instruction. +/* Returns a byte offset into the last executed instruction. * Does not raise an exception. */ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); diff --git a/Include/internal/mimalloc/mimalloc/atomic.h b/Include/internal/mimalloc/mimalloc/atomic.h index eb8478ceed6adf..52f82487685cdb 100644 --- a/Include/internal/mimalloc/mimalloc/atomic.h +++ b/Include/internal/mimalloc/mimalloc/atomic.h @@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file // -------------------------------------------------------------------------------------------- // Atomics // We need to be portable between C, C++, and MSVC. -// We base the primitives on the C/C++ atomics and create a mimimal wrapper for MSVC in C compilation mode. +// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode. // This is why we try to use only `uintptr_t` and `*` as atomic types. // To gain better insight in the range of used atomics, we use explicitly named memory order operations // instead of passing the memory order as a parameter. diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index 94f88fb603af25..d97f51b8eefbe5 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -10,7 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file // -------------------------------------------------------------------------- -// This file contains the interal API's of mimalloc and various utility +// This file contains the internal API's of mimalloc and various utility // functions and macros. // -------------------------------------------------------------------------- diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 1c68161341860a..6bf1b53bffd3ba 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -157,7 +157,7 @@ typedef enum { /* For now, this means the GIL is enabled. gh-116329: This will eventually change to "the GIL is disabled but can - be reenabled by loading an incompatible extension module." */ + be re-enabled by loading an incompatible extension module." */ _PyConfig_GIL_DEFAULT = -1, /* The GIL has been forced off or on, and will not be affected by module loading. */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index c98e82c8be5546..4e5b374968ea98 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -23,7 +23,7 @@ typedef uint32_t _PyMonitoringEventSet; #define PY_MONITORING_PROFILER_ID 2 #define PY_MONITORING_OPTIMIZER_ID 5 -/* Internal IDs used to suuport sys.setprofile() and sys.settrace() */ +/* Internal IDs used to support sys.setprofile() and sys.settrace() */ #define PY_MONITORING_SYS_PROFILE_ID 6 #define PY_MONITORING_SYS_TRACE_ID 7 diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index d5853b2c9ff464..882c4888e5058c 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -116,7 +116,7 @@ typedef enum _PyLockFlags { extern PyLockStatus _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout_ns, _PyLockFlags flags); -// Lock a mutex with aditional options. See _PyLockFlags for details. +// Lock a mutex with additional options. See _PyLockFlags for details. static inline void PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) { diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 3610c6254db6af..f3f5942444e851 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -147,7 +147,7 @@ PyAPI_FUNC(int) PyThread_start_joinable_thread(void (*func)(void *), PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t); /* * Detach a thread started with `PyThread_start_joinable_thread`, such - * that its resources are relased as soon as it exits. + * that its resources are released as soon as it exits. * This function cannot be interrupted. It returns 0 on success, * a non-zero value on failure. */ diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index fea5ceea0954f4..026d6e461f2108 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -189,7 +189,7 @@ extern PyObject* _PyUnicode_EncodeCharmap( /* --- Decimal Encoder ---------------------------------------------------- */ -// Coverts a Unicode object holding a decimal value to an ASCII string +// Converts a Unicode object holding a decimal value to an ASCII string // for using in int, float and complex parsers. // Transforms code points that have decimal digit property to the // corresponding ASCII digit code points. Transforms spaces to ASCII. diff --git a/Include/structmember.h b/Include/structmember.h index f6e8fd829892f4..5f29fbcfed99e3 100644 --- a/Include/structmember.h +++ b/Include/structmember.h @@ -11,7 +11,7 @@ extern "C" { * New definitions are in descrobject.h. * * However, there's nothing wrong with old code continuing to use it, - * and there's not much mainenance overhead in maintaining a few aliases. + * and there's not much maintenance overhead in maintaining a few aliases. * So, don't be too eager to convert old code. * * It uses names not prefixed with Py_. From 47816f465e833a5257a82b759b1081e06381e528 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Fri, 7 Jun 2024 10:38:15 +0200 Subject: [PATCH 407/903] gh-120154: Fix Emscripten/WASI pattern in case statement for LDSHARED (#120173) Fix Emscripten/WASI pattern in case statement for LDSHARED --- configure | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 6cfe114fb2104c..8e605d31bb5eca 100755 --- a/configure +++ b/configure @@ -12892,7 +12892,7 @@ then LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' BLDSHARED="$LDSHARED" ;; - Emscripten|WASI) + Emscripten*|WASI*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; Linux*|GNU*|QNX*|VxWorks*|Haiku*) diff --git a/configure.ac b/configure.ac index 8657e09c9a7008..41023ab92bad81 100644 --- a/configure.ac +++ b/configure.ac @@ -3417,7 +3417,7 @@ then LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' BLDSHARED="$LDSHARED" ;; - Emscripten|WASI) + Emscripten*|WASI*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; Linux*|GNU*|QNX*|VxWorks*|Haiku*) From d5ba4fc9bc9b2d9eff2a90893e8d500e0c367237 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Fri, 7 Jun 2024 12:14:13 +0300 Subject: [PATCH 408/903] gh-120164: Fix test_os.test_win32_mkdir_700() (#120177) Don't compare the path to avoid encoding issues. Co-authored-by: Eryk Sun --- Lib/test/test_os.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index de5a86f676c4d5..2beb9ca8aa6ccb 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1837,9 +1837,10 @@ def test_win32_mkdir_700(self): os.mkdir(path, mode=0o700) out = subprocess.check_output(["cacls.exe", path, "/s"], encoding="oem") os.rmdir(path) + out = out.strip().rsplit(" ", 1)[1] self.assertEqual( - out.strip(), - f'{path} "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"', + out, + '"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"', ) def tearDown(self): From 6646a9da26d12fc54263b22dd2916a2f710f1db7 Mon Sep 17 00:00:00 2001 From: Aditya Borikar Date: Fri, 7 Jun 2024 03:44:42 -0600 Subject: [PATCH 409/903] gh-110383: Clarify "non-integral" wording in pow() docs (#119688) --- Doc/library/functions.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 4617767a71be18..1d82f92ea67857 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1561,7 +1561,9 @@ are always available. They are listed here in alphabetical order. returns ``100``, but ``pow(10, -2)`` returns ``0.01``. For a negative base of type :class:`int` or :class:`float` and a non-integral exponent, a complex result is delivered. For example, ``pow(-9, 0.5)`` returns a value close - to ``3j``. + to ``3j``. Whereas, for a negative base of type :class:`int` or :class:`float` + with an integral exponent, a float result is delivered. For example, + ``pow(-9, 2.0)`` returns ``81.0``. For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must also be of integer type and *mod* must be nonzero. If *mod* is present and From d68a22e7a68ae09f7db61d5a1a3bd9c0360cf3ee Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 7 Jun 2024 13:49:07 +0300 Subject: [PATCH 410/903] gh-120211: Fix tkinter.ttk with Tcl/Tk 9.0 (GH-120213) * Use new methods for tracing Tcl variable. * Fix Combobox.current() for empty combobox. --- Lib/tkinter/ttk.py | 9 ++++++--- .../2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index 5ca938a670831a..073b3ae20797c3 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -690,7 +690,10 @@ def current(self, newindex=None): returns the index of the current value in the list of values or -1 if the current value does not appear in the list.""" if newindex is None: - return self.tk.getint(self.tk.call(self._w, "current")) + res = self.tk.call(self._w, "current") + if res == '': + return -1 + return self.tk.getint(res) return self.tk.call(self._w, "current", newindex) @@ -1522,7 +1525,7 @@ def __init__(self, master=None, variable=None, from_=0, to=10, **kw): self.label.place(anchor='n' if label_side == 'top' else 's') # update the label as scale or variable changes - self.__tracecb = self._variable.trace_variable('w', self._adjust) + self.__tracecb = self._variable.trace_add('write', self._adjust) self.bind('', self._adjust) self.bind('', self._adjust) @@ -1530,7 +1533,7 @@ def __init__(self, master=None, variable=None, from_=0, to=10, **kw): def destroy(self): """Destroy this widget and possibly its associated variable.""" try: - self._variable.trace_vdelete('w', self.__tracecb) + self._variable.trace_remove('write', self.__tracecb) except AttributeError: pass else: diff --git a/Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst b/Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst new file mode 100644 index 00000000000000..0106f2d93318b4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-07-13-21-11.gh-issue-120211.Rws_gf.rst @@ -0,0 +1 @@ +Fix :mod:`tkinter.ttk` with Tcl/Tk 9.0. From eca3f7762c23b22a73a5e0b09520748c88aab4a0 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 7 Jun 2024 14:06:24 +0100 Subject: [PATCH 411/903] gh-93691: fix too broad source locations of with-statement instructions (#120125) --- Lib/test/test_with.py | 44 +++++++++++++++++++ ...4-06-05-18-29-18.gh-issue-93691.6OautB.rst | 1 + Python/compile.c | 5 +-- 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index d81902327a7e0a..e8c4ddf979e2ee 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -5,6 +5,7 @@ __email__ = "mbland at acm dot org" import sys +import traceback import unittest from collections import deque from contextlib import _GeneratorContextManager, contextmanager, nullcontext @@ -749,5 +750,48 @@ def testEnterReturnsTuple(self): self.assertEqual(10, b1) self.assertEqual(20, b2) + def testExceptionLocation(self): + # The location of an exception raised from + # __init__, __enter__ or __exit__ of a context + # manager should be just the context manager expression, + # pinpointing the precise context manager in case there + # is more than one. + + def init_raises(): + try: + with self.Dummy(), self.InitRaises() as cm, self.Dummy() as d: + pass + except Exception as e: + return e + + def enter_raises(): + try: + with self.EnterRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + def exit_raises(): + try: + with self.ExitRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "self.InitRaises()"), + (enter_raises, "self.EnterRaises()"), + (exit_raises, "self.ExitRaises()"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst new file mode 100644 index 00000000000000..c06d5a276c03eb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-18-29-18.gh-issue-93691.6OautB.rst @@ -0,0 +1 @@ +Fix source locations of instructions generated for with statements. diff --git a/Python/compile.c b/Python/compile.c index 7d74096fcdf94e..cb724154206b7e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5900,7 +5900,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* Evaluate EXPR */ VISIT(c, expr, item->context_expr); - + loc = LOC(item->context_expr); ADDOP(c, loc, BEFORE_ASYNC_WITH); ADDOP_I(c, loc, GET_AWAITABLE, 1); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -5998,7 +5998,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) /* Evaluate EXPR */ VISIT(c, expr, item->context_expr); /* Will push bound __exit__ */ - location loc = LOC(s); + location loc = LOC(item->context_expr); ADDOP(c, loc, BEFORE_WITH); ADDOP_JUMP(c, loc, SETUP_WITH, final); @@ -6031,7 +6031,6 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) /* For successful outcome: * call __exit__(None, None, None) */ - loc = LOC(s); RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); ADDOP(c, loc, POP_TOP); ADDOP_JUMP(c, loc, JUMP, exit); From 225aab7f70d804174cc3a75bc04a5bb1545e5adb Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 7 Jun 2024 15:37:18 +0200 Subject: [PATCH 412/903] gh-110383: Improve 'old string formatting' text in tutorial (#120219) --- Doc/tutorial/inputoutput.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 857068a51ab843..b93a0e8cec2d38 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -279,9 +279,11 @@ left with zeros. It understands about plus and minus signs:: Old string formatting --------------------- -The % operator (modulo) can also be used for string formatting. Given ``'string' -% values``, instances of ``%`` in ``string`` are replaced with zero or more -elements of ``values``. This operation is commonly known as string +The % operator (modulo) can also be used for string formatting. +Given ``format % values`` (where *format* is a string), +``%`` conversion specifications in *format* are replaced with +zero or more elements of *values*. +This operation is commonly known as string interpolation. For example:: >>> import math From 9d6604222e9ef4e136ee9ccfa2d4d5ff9feee976 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 7 Jun 2024 17:42:01 +0200 Subject: [PATCH 413/903] gh-114264: Optimize performance of copy.deepcopy by adding a fast path for atomic types (GH-114266) --- Lib/copy.py | 31 ++++++------------- ...-01-18-21-44-23.gh-issue-114264.DBKn29.rst | 1 + 2 files changed, 11 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst diff --git a/Lib/copy.py b/Lib/copy.py index a69bc4e78c20b3..7a1907d75494d7 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -121,6 +121,11 @@ def deepcopy(x, memo=None, _nil=[]): See the module's __doc__ string for more info. """ + cls = type(x) + + if cls in _atomic_types: + return x + d = id(x) if memo is None: memo = {} @@ -129,14 +134,12 @@ def deepcopy(x, memo=None, _nil=[]): if y is not _nil: return y - cls = type(x) - copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x, memo) else: if issubclass(cls, type): - y = _deepcopy_atomic(x, memo) + y = x # atomic copy else: copier = getattr(x, "__deepcopy__", None) if copier is not None: @@ -167,26 +170,12 @@ def deepcopy(x, memo=None, _nil=[]): _keep_alive(x, memo) # Make sure x lives at least as long as d return y +_atomic_types = {types.NoneType, types.EllipsisType, types.NotImplementedType, + int, float, bool, complex, bytes, str, types.CodeType, type, range, + types.BuiltinFunctionType, types.FunctionType, weakref.ref, property} + _deepcopy_dispatch = d = {} -def _deepcopy_atomic(x, memo): - return x -d[types.NoneType] = _deepcopy_atomic -d[types.EllipsisType] = _deepcopy_atomic -d[types.NotImplementedType] = _deepcopy_atomic -d[int] = _deepcopy_atomic -d[float] = _deepcopy_atomic -d[bool] = _deepcopy_atomic -d[complex] = _deepcopy_atomic -d[bytes] = _deepcopy_atomic -d[str] = _deepcopy_atomic -d[types.CodeType] = _deepcopy_atomic -d[type] = _deepcopy_atomic -d[range] = _deepcopy_atomic -d[types.BuiltinFunctionType] = _deepcopy_atomic -d[types.FunctionType] = _deepcopy_atomic -d[weakref.ref] = _deepcopy_atomic -d[property] = _deepcopy_atomic def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] diff --git a/Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst b/Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst new file mode 100644 index 00000000000000..069ac68b4f3a95 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-18-21-44-23.gh-issue-114264.DBKn29.rst @@ -0,0 +1 @@ +Improve performance of :func:`copy.deepcopy` by adding a fast path for atomic types. From 10fb1b8f36ab2fc3d2fe7392d5735dd19c5e2365 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 7 Jun 2024 18:48:31 +0300 Subject: [PATCH 414/903] gh-120200: Fix `inspect.iscoroutinefunction(inspect) is True` corner case (#120214) --- Lib/inspect.py | 6 +++--- Lib/test/test_inspect/test_inspect.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index e6e49a4ffa673a..2b7f8bec482f8e 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -403,13 +403,13 @@ def isgeneratorfunction(obj): return _has_code_flag(obj, CO_GENERATOR) # A marker for markcoroutinefunction and iscoroutinefunction. -_is_coroutine_marker = object() +_is_coroutine_mark = object() def _has_coroutine_mark(f): while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) - return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_marker + return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_mark def markcoroutinefunction(func): """ @@ -417,7 +417,7 @@ def markcoroutinefunction(func): """ if hasattr(func, '__func__'): func = func.__func__ - func._is_coroutine_marker = _is_coroutine_marker + func._is_coroutine_marker = _is_coroutine_mark return func def iscoroutinefunction(obj): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 65007c16203c6d..0a4fa9343f15e0 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -235,6 +235,7 @@ class PMClass: gen_coroutine_function_example)))) self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmi)) self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmc)) + self.assertFalse(inspect.iscoroutinefunction(inspect)) self.assertFalse(inspect.iscoroutine(gen_coro)) self.assertTrue( From a7584245661102a5768c643fbd7db8395fd3c90e Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Fri, 7 Jun 2024 23:51:32 +0800 Subject: [PATCH 415/903] gh-120226: Fix test_sendfile_close_peer_in_the_middle_of_receiving on Linux >= 6.10 (#120227) The worst case is that the kernel buffers 17 pages with a page size of 64k. --- Lib/test/test_asyncio/test_sendfile.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index d33ff197bbfa1d..2509d4382cdebd 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -93,13 +93,10 @@ async def wait_closed(self): class SendfileBase: - # 256 KiB plus small unaligned to buffer chunk - # Newer versions of Windows seems to have increased its internal - # buffer and tries to send as much of the data as it can as it - # has some form of buffering for this which is less than 256KiB - # on newer server versions and Windows 11. - # So DATA should be larger than 256 KiB to make this test reliable. - DATA = b"x" * (1024 * 256 + 1) + # Linux >= 6.10 seems buffering up to 17 pages of data. + # So DATA should be large enough to make this test reliable even with a + # 64 KiB page configuration. + DATA = b"x" * (1024 * 17 * 64 + 1) # Reduce socket buffer size to test on relative small data sets. BUF_SIZE = 4 * 1024 # 4 KiB From 90b75405260467814c93738a3325645918d4ea51 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 7 Jun 2024 17:58:21 +0200 Subject: [PATCH 416/903] gh-120155: Fix copy/paste error in HAVE_SUBOFFSETS_IN_LAST_DIM() (#120228) Don't hardcode 'dest' in HAVE_SUBOFFSETS_IN_LAST_DIM() macro of memoryobject.c, but use its 'view' parameter instead. Fix the Coverity issue: Error: COPY_PASTE_ERROR (CWE-398): Python-3.12.2/Objects/memoryobject.c:273:14: original: ""dest->suboffsets + (dest->ndim - 1)"" looks like the original copy. Python-3.12.2/Objects/memoryobject.c:274:14: copy_paste_error: ""dest"" in ""src->suboffsets + (dest->ndim - 1)"" looks like a copy-paste error. Python-3.12.2/Objects/memoryobject.c:274:14: remediation: Should it say ""src"" instead? # 272| assert(dest->ndim > 0 && src->ndim > 0); # 273| return (!HAVE_SUBOFFSETS_IN_LAST_DIM(dest) && # 274|-> !HAVE_SUBOFFSETS_IN_LAST_DIM(src) && # 275| dest->strides[dest->ndim-1] == dest->itemsize && # 276| src->strides[src->ndim-1] == src->itemsize); --- Objects/memoryobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 5caa6504272301..226bd6defdec5a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -268,7 +268,7 @@ PyTypeObject _PyManagedBuffer_Type = { /* Assumptions: ndim >= 1. The macro tests for a corner case that should perhaps be explicitly forbidden in the PEP. */ #define HAVE_SUBOFFSETS_IN_LAST_DIM(view) \ - (view->suboffsets && view->suboffsets[dest->ndim-1] >= 0) + (view->suboffsets && view->suboffsets[view->ndim-1] >= 0) static inline int last_dim_is_contiguous(const Py_buffer *dest, const Py_buffer *src) From 242c7498e5a889b47847fb6f0f133ce461fa7e24 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 7 Jun 2024 17:59:34 +0100 Subject: [PATCH 417/903] GH-116380: Move pathlib-specific code from `glob` to `pathlib._abc`. (#120011) In `glob._Globber`, move pathlib-specific methods to `pathlib._abc.PathGlobber` and replace them with abstract methods. Rename `glob._Globber` to `glob._GlobberBase`. As a result, the `glob` module is no longer befouled by code that can only ever apply to pathlib. No change of behaviour. --- Lib/glob.py | 34 ++++++++++++++++++++++------------ Lib/pathlib/_abc.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/Lib/glob.py b/Lib/glob.py index fbb1d35aab71fa..574e5ad51b601d 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -328,8 +328,8 @@ def _compile_pattern(pat, sep, case_sensitive, recursive=True): return re.compile(regex, flags=flags).match -class _Globber: - """Class providing shell-style pattern matching and globbing. +class _GlobberBase: + """Abstract class providing shell-style pattern matching and globbing. """ def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): @@ -338,29 +338,37 @@ def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): self.case_pedantic = case_pedantic self.recursive = recursive - # Low-level methods + # Abstract methods - lexists = operator.methodcaller('exists', follow_symlinks=False) - add_slash = operator.methodcaller('joinpath', '') + @staticmethod + def lexists(path): + """Implements os.path.lexists(). + """ + raise NotImplementedError @staticmethod def scandir(path): - """Emulates os.scandir(), which returns an object that can be used as - a context manager. This method is called by walk() and glob(). + """Implements os.scandir(). + """ + raise NotImplementedError + + @staticmethod + def add_slash(path): + """Returns a path with a trailing slash added. """ - return contextlib.nullcontext(path.iterdir()) + raise NotImplementedError @staticmethod def concat_path(path, text): - """Appends text to the given path. + """Implements path concatenation. """ - return path.with_segments(path._raw_path + text) + raise NotImplementedError @staticmethod def parse_entry(entry): """Returns the path of an entry yielded from scandir(). """ - return entry + raise NotImplementedError # High-level methods @@ -520,7 +528,9 @@ def select_exists(self, path, exists=False): yield path -class _StringGlobber(_Globber): +class _StringGlobber(_GlobberBase): + """Provides shell-style pattern matching and globbing for string paths. + """ lexists = staticmethod(os.path.lexists) scandir = staticmethod(os.scandir) parse_entry = operator.attrgetter('path') diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 1a74f457c3f5a7..ecea8e88d1a2e3 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -12,8 +12,9 @@ """ import functools +import operator import posixpath -from glob import _Globber, _no_recurse_symlinks +from glob import _GlobberBase, _no_recurse_symlinks from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO @@ -84,6 +85,33 @@ def isabs(self, path): raise UnsupportedOperation(self._unsupported_msg('isabs()')) +class PathGlobber(_GlobberBase): + """ + Class providing shell-style globbing for path objects. + """ + + lexists = operator.methodcaller('exists', follow_symlinks=False) + add_slash = operator.methodcaller('joinpath', '') + + @staticmethod + def scandir(path): + """Emulates os.scandir(), which returns an object that can be used as + a context manager. This method is called by walk() and glob(). + """ + import contextlib + return contextlib.nullcontext(path.iterdir()) + + @staticmethod + def concat_path(path, text): + """Appends text to the given path.""" + return path.with_segments(path._raw_path + text) + + @staticmethod + def parse_entry(entry): + """Returns the path of an entry yielded from scandir().""" + return entry + + class PurePathBase: """Base class for pure path objects. @@ -104,7 +132,7 @@ class PurePathBase: '_resolving', ) parser = ParserBase() - _globber = _Globber + _globber = PathGlobber def __init__(self, path, *paths): self._raw_path = self.parser.join(path, *paths) if paths else path From e6076d1e1303c3cc14bc02baf607535af2cf1501 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 7 Jun 2024 13:44:56 -0400 Subject: [PATCH 418/903] gh-119659: Get the datetime CAPI Tests Running Again (gh-120180) The tests were accidentally disabled by 2da0dc0, which didn't handle classes correctly. I considered updating no_rerun() to support classes, but the way test_datetime.py works would have made things fairly messy. Plus, it looks like the refleaks we had encountered before have been resolved. --- Lib/test/datetimetester.py | 3 +-- Lib/test/support/__init__.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 3759504b02e550..b80da5697ef865 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -22,7 +22,7 @@ from test import support from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST -from test.support import warnings_helper, no_rerun +from test.support import warnings_helper import datetime as datetime_module from datetime import MINYEAR, MAXYEAR @@ -6385,7 +6385,6 @@ class IranTest(ZoneInfoTest): @unittest.skipIf(_testcapi is None, 'need _testcapi module') -@no_rerun("the encapsulated datetime C API does not support reloading") class CapiTest(unittest.TestCase): def setUp(self): # Since the C API is not present in the _Pure tests, skip all tests diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 4b320b494bb8dd..9e6100d2b89d6e 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1197,6 +1197,7 @@ def no_rerun(reason): test using the 'reason' parameter. """ def deco(func): + assert not isinstance(func, type), func _has_run = False def wrapper(self): nonlocal _has_run From 4fc82b6d3b99f873179937215833e7a573ca7876 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 7 Jun 2024 22:37:35 +0100 Subject: [PATCH 419/903] gh-120225: fix crash in compiler on empty block at end of exception handler (#120235) --- Lib/test/test_compile.py | 10 ++++++++++ .../2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst | 1 + Python/flowgraph.c | 8 ++------ 3 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index ba0bcc9c1ced99..ae23aea08d99bc 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1409,6 +1409,16 @@ def f(): for kw in ("except", "except*"): exec(code % kw, g, l); + def test_regression_gh_120225(self): + async def name_4(): + match b'': + case True: + pass + case name_5 if f'e': + {name_3: name_4 async for name_2 in name_5} + case []: + pass + [[]] @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst new file mode 100644 index 00000000000000..d00b9aaa8192e3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-07-16-09-04.gh-issue-120225.kuYf9t.rst @@ -0,0 +1 @@ +Fix crash in compiler on empty block at end of exception handler. diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 17617e119fdaa4..aed694aee91f91 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2304,15 +2304,11 @@ push_cold_blocks_to_end(cfg_builder *g) { if (!IS_LABEL(b->b_next->b_label)) { b->b_next->b_label.id = next_lbl++; } - cfg_instr *prev_instr = basicblock_last_instr(b); - // b cannot be empty because at the end of an exception handler - // there is always a POP_EXCEPT + RERAISE/RETURN - assert(prev_instr); - basicblock_addop(explicit_jump, JUMP_NO_INTERRUPT, b->b_next->b_label.id, - prev_instr->i_loc); + NO_LOCATION); explicit_jump->b_cold = 1; explicit_jump->b_next = b->b_next; + explicit_jump->b_predecessors = 1; b->b_next = explicit_jump; /* set target */ From 95f4db88d5ab7d900f05d0418b2a2e77bf9ff126 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 8 Jun 2024 10:51:09 +0300 Subject: [PATCH 420/903] gh-120242: Fix handling of `[setUp,tearDown]Class` in `test_datetime` (#120243) --- Lib/test/test_datetime.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 3859733a4fe65b..005187f13e665f 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -1,5 +1,6 @@ import unittest import sys +import functools from test.support.import_helper import import_fresh_module @@ -39,21 +40,26 @@ def load_tests(loader, tests, pattern): for cls in test_classes: cls.__name__ += suffix cls.__qualname__ += suffix - @classmethod - def setUpClass(cls_, module=module): - cls_._save_sys_modules = sys.modules.copy() - sys.modules[TESTS] = module - sys.modules['datetime'] = module.datetime_module - if hasattr(module, '_pydatetime'): - sys.modules['_pydatetime'] = module._pydatetime - sys.modules['_strptime'] = module._strptime - @classmethod - def tearDownClass(cls_): - sys.modules.clear() - sys.modules.update(cls_._save_sys_modules) - cls.setUpClass = setUpClass - cls.tearDownClass = tearDownClass - tests.addTests(loader.loadTestsFromTestCase(cls)) + + @functools.wraps(cls, updated=()) + class Wrapper(cls): + @classmethod + def setUpClass(cls_, module=module): + cls_._save_sys_modules = sys.modules.copy() + sys.modules[TESTS] = module + sys.modules['datetime'] = module.datetime_module + if hasattr(module, '_pydatetime'): + sys.modules['_pydatetime'] = module._pydatetime + sys.modules['_strptime'] = module._strptime + super().setUpClass() + + @classmethod + def tearDownClass(cls_): + super().tearDownClass() + sys.modules.clear() + sys.modules.update(cls_._save_sys_modules) + + tests.addTests(loader.loadTestsFromTestCase(Wrapper)) return tests From 2080425154d235b4b7dcc9a8a2f58e71769125ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrico=20Tr=C3=B6ger?= Date: Sat, 8 Jun 2024 11:19:13 +0200 Subject: [PATCH 421/903] bpo-37755: Use configured output in pydoc instead of pager (GH-15105) If the Helper() class was initialized with an output, the topics, keywords and symbols help still use the pager instead of the output. Change the behavior so the output is used if available while keeping the previous behavior if no output was configured. --- Lib/pydoc.py | 8 +- Lib/test/test_pydoc/test_pydoc.py | 125 +++++++++++++++--- ...4-06-02-13-35-11.gh-issue-81936.ETeW9x.rst | 3 + 3 files changed, 116 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 2ba597d01f245e..d7579c1cc3dcd1 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -2034,7 +2034,7 @@ def help(self, request, is_cli=False): elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too - doc(eval(request), 'Help on %s:', is_cli=is_cli) + doc(eval(request), 'Help on %s:', output=self._output, is_cli=is_cli) elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) @@ -2127,7 +2127,11 @@ def showtopic(self, topic, more_xrefs=''): text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) - pager(doc, f'Help on {topic!s}') + + if self._output is None: + pager(doc, f'Help on {topic!s}') + else: + self.output.write(doc) def _gettopic(self, topic, more_xrefs=''): """Return unbuffered tuple of (topic, xrefs). diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 436fdb38756ddd..57e5b8e8abddfa 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -17,6 +17,7 @@ import types import typing import unittest +import unittest.mock import urllib.parse import xml.etree import xml.etree.ElementTree @@ -658,16 +659,13 @@ def test_fail_help_output_redirect(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') + @unittest.mock.patch('pydoc.pager') @requires_docstrings - def test_help_output_redirect(self): + def test_help_output_redirect(self, pager_mock): # issue 940286, if output is set in Helper, then all output from # Helper.help should be redirected - getpager_old = pydoc.getpager - getpager_new = lambda: (lambda x: x) self.maxDiff = None - buf = StringIO() - helper = pydoc.Helper(output=buf) unused, doc_loc = get_pydoc_text(pydoc_mod) module = "test.test_pydoc.pydoc_mod" help_header = """ @@ -677,21 +675,112 @@ def test_help_output_redirect(self): help_header = textwrap.dedent(help_header) expected_help_pattern = help_header + expected_text_pattern - pydoc.getpager = getpager_new - try: + with captured_output('stdout') as output, \ + captured_output('stderr') as err, \ + StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.help(module) + result = buf.getvalue().strip() + expected_text = expected_help_pattern % ( + (doc_loc,) + + expected_text_data_docstrings + + (inspect.getabsfile(pydoc_mod),)) + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertEqual(expected_text, result) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_help_output_redirect_various_requests(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.help should be redirected + + def run_pydoc_for_request(request, expected_text_part): + """Helper function to run pydoc with its output redirected""" with captured_output('stdout') as output, \ - captured_output('stderr') as err: - helper.help(module) + captured_output('stderr') as err, \ + StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.help(request) result = buf.getvalue().strip() - expected_text = expected_help_pattern % ( - (doc_loc,) + - expected_text_data_docstrings + - (inspect.getabsfile(pydoc_mod),)) - self.assertEqual('', output.getvalue()) - self.assertEqual('', err.getvalue()) - self.assertEqual(expected_text, result) - finally: - pydoc.getpager = getpager_old + self.assertEqual('', output.getvalue(), msg=f'failed on request "{request}"') + self.assertEqual('', err.getvalue(), msg=f'failed on request "{request}"') + self.assertIn(expected_text_part, result, msg=f'failed on request "{request}"') + pager_mock.assert_not_called() + + self.maxDiff = None + + # test for "keywords" + run_pydoc_for_request('keywords', 'Here is a list of the Python keywords.') + # test for "symbols" + run_pydoc_for_request('symbols', 'Here is a list of the punctuation symbols') + # test for "topics" + run_pydoc_for_request('topics', 'Here is a list of available topics.') + # test for "modules" skipped, see test_modules() + # test for symbol "%" + run_pydoc_for_request('%', 'The power operator') + # test for special True, False, None keywords + run_pydoc_for_request('True', 'class bool(int)') + run_pydoc_for_request('False', 'class bool(int)') + run_pydoc_for_request('None', 'class NoneType(object)') + # test for keyword "assert" + run_pydoc_for_request('assert', 'The "assert" statement') + # test for topic "TYPES" + run_pydoc_for_request('TYPES', 'The standard type hierarchy') + # test for "pydoc.Helper.help" + run_pydoc_for_request('pydoc.Helper.help', 'Help on function help in pydoc.Helper:') + # test for pydoc.Helper.help + run_pydoc_for_request(pydoc.Helper.help, 'Help on function help in module pydoc:') + # test for pydoc.Helper() instance skipped because it is always meant to be interactive + + def test_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('with') + helptext = showtopic_io.getvalue() + self.assertIn('The "with" statement', helptext) + + def test_fail_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('abd') + expected = "no documentation found for 'abd'" + self.assertEqual(expected, showtopic_io.getvalue().strip()) + + @unittest.mock.patch('pydoc.pager') + def test_fail_showtopic_output_redirect(self, pager_mock): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.showtopic("abd") + expected = "no documentation found for 'abd'" + self.assertEqual(expected, buf.getvalue().strip()) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_showtopic_output_redirect(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.showtopic should be redirected + self.maxDiff = None + + with captured_output('stdout') as output, \ + captured_output('stderr') as err, \ + StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.showtopic('with') + result = buf.getvalue().strip() + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertIn('The "with" statement', result) + + pager_mock.assert_not_called() def test_lambda_with_return_annotation(self): func = lambda a, b, c: 1 diff --git a/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst b/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst new file mode 100644 index 00000000000000..d53cc73e728d54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst @@ -0,0 +1,3 @@ +:meth:`!help` and :meth:`!showtopic` methods now respect a +configured *output* argument to :class:`!pydoc.Helper` and not use the +pager in such cases. Patch by Enrico Tröger. From 55402d3232ca400ebafe4fe3bd70f252304ebe07 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Sat, 8 Jun 2024 05:41:45 -0400 Subject: [PATCH 422/903] gh-119258: Eliminate Type Guards in Tier 2 Optimizer with Watcher (GH-119365) Co-authored-by: parmeggiani Co-authored-by: dpdani Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Brandt Bucher Co-authored-by: Ken Jin --- Include/internal/pycore_optimizer.h | 9 +- Include/internal/pycore_typeobject.h | 11 ++ Lib/test/test_capi/test_opt.py | 147 ++++++++++++++++++ Lib/test/test_capi/test_watchers.py | 6 +- Lib/test/test_type_cache.py | 3 +- ...-05-23-20-17-37.gh-issue-119258.wZFIpt.rst | 3 + Modules/_testcapimodule.c | 17 -- Modules/_testinternalcapi.c | 19 +++ Objects/typeobject.c | 78 ++++++++-- Python/optimizer_analysis.c | 16 +- Python/optimizer_bytecodes.c | 37 ++++- Python/optimizer_cases.c.h | 33 +++- Python/optimizer_symbols.c | 46 +++++- 13 files changed, 366 insertions(+), 59 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 76123987ac99f5..fd7833fd231299 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -33,6 +33,7 @@ struct _Py_UopsSymbol { int flags; // 0 bits: Top; 2 or more bits: Bottom PyTypeObject *typ; // Borrowed reference PyObject *const_val; // Owned reference (!) + unsigned int type_version; // currently stores type version }; #define UOP_FORMAT_TARGET 0 @@ -123,9 +124,11 @@ extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *con extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym); extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version); extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version); extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val); extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym); @@ -138,9 +141,9 @@ extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( _Py_UOpsContext *ctx, PyCodeObject *co, - _Py_UopsSymbol **localsplus_start, - int n_locals_already_filled, - int curr_stackentries); + int curr_stackentries, + _Py_UopsSymbol **args, + int arg_len); extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 8664ae0e44533f..bc295b1b066bd1 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -63,6 +63,8 @@ typedef struct { PyObject *tp_weaklist; } managed_static_type_state; +#define TYPE_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ + struct types_state { /* Used to set PyTypeObject.tp_version_tag. It starts at _Py_MAX_GLOBAL_TYPE_VERSION_TAG + 1, @@ -118,6 +120,12 @@ struct types_state { managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_EXT_TYPES]; } for_extensions; PyMutex mutex; + + // Borrowed references to type objects whose + // tp_version_tag % TYPE_VERSION_CACHE_SIZE + // once was equal to the index in the table. + // They are cleared when the type object is deallocated. + PyTypeObject *type_version_cache[TYPE_VERSION_CACHE_SIZE]; }; @@ -230,6 +238,9 @@ extern void _PyType_SetFlags(PyTypeObject *self, unsigned long mask, extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags); +extern unsigned int _PyType_GetVersionForCurrentState(PyTypeObject *tp); +PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version); +PyTypeObject *_PyType_LookupByVersion(unsigned int version); #ifdef __cplusplus } diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 0491ff9b84d486..fc6d8b0a3f01d2 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1333,6 +1333,153 @@ def test_modified_local_is_seen_by_optimized_code(self): self.assertIs(type(s), float) self.assertEqual(s, 1024.0) + def test_guard_type_version_removed(self): + def thing(a): + x = 0 + for _ in range(100): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_removed_inlined(self): + """ + Verify that the guard type version if we have an inlined function + """ + + def fn(): + pass + + def thing(a): + x = 0 + for _ in range(100): + x += a.attr + fn() + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_not_removed(self): + """ + Verify that the guard type version is not removed if we modify the class + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + # for the first 90 iterations we set the attribute on this dummy function which shouldn't + # trigger the type watcher + # then after 90 it should trigger it and stop optimizing + # Note that the code needs to be in this weird form so it's optimized inline without any control flow + setattr((Foo, Bar)[i < 90], "attr", 2) + x += a.attr + return x + + class Foo: + attr = 1 + + class Bar: + pass + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + + self.assertIsNotNone(ex) + self.assertEqual(res, 219) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 2) + + + @unittest.expectedFailure + def test_guard_type_version_not_removed_escaping(self): + """ + Verify that the guard type version is not removed if have an escaping function + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + # eval should be escaping and so should cause optimization to stop and preserve both type versions + eval("None") + x += a.attr + return x + + class Foo: + attr = 1 + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + # Note: This will actually be 1 for noe + # https://github.com/python/cpython/pull/119365#discussion_r1626220129 + self.assertEqual(guard_type_version_count, 2) + + + def test_guard_type_version_executor_invalidated(self): + """ + Verify that the executor is invalided on a type change. + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + self.assertEqual(res, 200) + self.assertIsNotNone(ex) + self.assertEqual(list(iter_opnames(ex)).count("_GUARD_TYPE_VERSION"), 1) + self.assertTrue(ex.is_valid()) + Foo.attr = 0 + self.assertFalse(ex.is_valid()) + + def test_type_version_doesnt_segfault(self): + """ + Tests that setting a type version doesn't cause a segfault when later looking at the stack. + """ + + # Minimized from mdp.py benchmark + + class A: + def __init__(self): + self.attr = {} + + def method(self, arg): + self.attr[arg] = None + + def fn(a): + for _ in range(100): + (_ for _ in []) + (_ for _ in [a.method(None)]) + + fn(A()) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 90665a7561b316..709b5e1c4b716a 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -282,8 +282,10 @@ class C: pass self.watch(wid, C) with catch_unraisable_exception() as cm: C.foo = "bar" - self.assertEqual(cm.unraisable.err_msg, - f"Exception ignored in type watcher callback #0 for {C!r}") + self.assertEqual( + cm.unraisable.err_msg, + f"Exception ignored in type watcher callback #1 for {C!r}", + ) self.assertIs(cm.unraisable.object, None) self.assertEqual(str(cm.unraisable.exc_value), "boom!") self.assert_events([]) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index e90e315c808361..edaf076707ad8b 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -10,8 +10,9 @@ # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module("_testcapi") +_testinternalcapi = import_helper.import_module("_testinternalcapi") type_get_version = _testcapi.type_get_version -type_assign_specific_version_unsafe = _testcapi.type_assign_specific_version_unsafe +type_assign_specific_version_unsafe = _testinternalcapi.type_assign_specific_version_unsafe type_assign_version = _testcapi.type_assign_version type_modified = _testcapi.type_modified diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst new file mode 100644 index 00000000000000..68f1ec1efa5751 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst @@ -0,0 +1,3 @@ +Eliminate type version guards in the tier two interpreter. + +Note that setting the ``tp_version_tag`` manually (which has never been supported) may result in crashes. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b58c17260626c2..b139b46c826a3f 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2403,21 +2403,6 @@ type_modified(PyObject *self, PyObject *type) Py_RETURN_NONE; } -// Circumvents standard version assignment machinery - use with caution and only on -// short-lived heap types -static PyObject * -type_assign_specific_version_unsafe(PyObject *self, PyObject *args) -{ - PyTypeObject *type; - unsigned int version; - if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) { - return NULL; - } - assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)); - type->tp_version_tag = version; - type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; - Py_RETURN_NONE; -} static PyObject * type_assign_version(PyObject *self, PyObject *type) @@ -3427,8 +3412,6 @@ static PyMethodDef TestMethods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, {"type_modified", type_modified, METH_O, PyDoc_STR("PyType_Modified")}, - {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, - PyDoc_STR("forcefully assign type->tp_version_tag")}, {"type_assign_version", type_assign_version, METH_O, PyDoc_STR("PyUnstable_Type_AssignVersionTag")}, {"type_get_tp_bases", type_get_tp_bases, METH_O}, {"type_get_tp_mro", type_get_tp_mro, METH_O}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 6d4a00c06ca9de..139a0509795de9 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2002,6 +2002,22 @@ has_inline_values(PyObject *self, PyObject *obj) } +// Circumvents standard version assignment machinery - use with caution and only on +// short-lived heap types +static PyObject * +type_assign_specific_version_unsafe(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + unsigned int version; + if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) { + return NULL; + } + assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)); + _PyType_SetVersion(type, version); + type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; + Py_RETURN_NONE; +} + /*[clinic input] gh_119213_getargs @@ -2102,6 +2118,9 @@ static PyMethodDef module_functions[] = { {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, {"has_inline_values", has_inline_values, METH_O}, + {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, + PyDoc_STR("forcefully assign type->tp_version_tag")}, + #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 880ac6b9c009fe..cd16bebd1e1cb8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -853,7 +853,8 @@ PyType_AddWatcher(PyType_WatchCallback callback) { PyInterpreterState *interp = _PyInterpreterState_GET(); - for (int i = 0; i < TYPE_MAX_WATCHERS; i++) { + // start at 1, 0 is reserved for cpython optimizer + for (int i = 1; i < TYPE_MAX_WATCHERS; i++) { if (!interp->type_watchers[i]) { interp->type_watchers[i] = callback; return i; @@ -960,7 +961,7 @@ type_modification_starting_unlocked(PyTypeObject *type) } /* 0 is not a valid version tag */ - _Py_atomic_store_uint32_release(&type->tp_version_tag, 0); + _PyType_SetVersion(type, 0); } #endif @@ -1024,7 +1025,7 @@ type_modified_unlocked(PyTypeObject *type) } type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */ + _PyType_SetVersion(type, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -1101,7 +1102,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { clear: assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */ + _PyType_SetVersion(type, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -1109,6 +1110,64 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { } } +/* +The Tier 2 interpreter requires looking up the type object by the type version, so it can install +watchers to understand when they change. + +So we add a global cache from type version to borrowed references of type objects. + +This is similar to func_version_cache. +*/ + +void +_PyType_SetVersion(PyTypeObject *tp, unsigned int version) +{ +#ifndef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + // lookup the old version and set to null + if (tp->tp_version_tag != 0) { + PyTypeObject **slot = + interp->types.type_version_cache + + (tp->tp_version_tag % TYPE_VERSION_CACHE_SIZE); + *slot = NULL; + } +#endif + FT_ATOMIC_STORE_UINT32_RELAXED(tp->tp_version_tag, version); +#ifndef Py_GIL_DISABLED + if (version != 0) { + PyTypeObject **slot = + interp->types.type_version_cache + + (version % TYPE_VERSION_CACHE_SIZE); + *slot = tp; + } +#endif +} + +PyTypeObject * +_PyType_LookupByVersion(unsigned int version) +{ +#ifdef Py_GIL_DISABLED + return NULL; +#else + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyTypeObject **slot = + interp->types.type_version_cache + + (version % TYPE_VERSION_CACHE_SIZE); + if (*slot && (*slot)->tp_version_tag == version) { + return *slot; + } + return NULL; +#endif +} + +unsigned int +_PyType_GetVersionForCurrentState(PyTypeObject *tp) +{ + return tp->tp_version_tag; +} + + + #define MAX_VERSIONS_PER_CLASS 1000 static int @@ -1137,8 +1196,7 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, - NEXT_GLOBAL_VERSION_TAG++); + _PyType_SetVersion(type, NEXT_GLOBAL_VERSION_TAG++); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -1147,8 +1205,7 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, - NEXT_VERSION_TAG(interp)++); + _PyType_SetVersion(type, NEXT_VERSION_TAG(interp)++); assert (type->tp_version_tag != 0); } @@ -5768,7 +5825,7 @@ fini_static_type(PyInterpreterState *interp, PyTypeObject *type, if (final) { type->tp_flags &= ~Py_TPFLAGS_READY; type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; + _PyType_SetVersion(type, 0); } _PyStaticType_ClearWeakRefs(interp, type); @@ -5798,7 +5855,6 @@ type_dealloc(PyObject *self) _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE); _PyObject_GC_UNTRACK(type); - type_dealloc_common(type); // PyObject_ClearWeakRefs() raises an exception if Py_REFCNT() != 0 @@ -8367,7 +8423,7 @@ init_static_type(PyInterpreterState *interp, PyTypeObject *self, self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); - self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; + _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; } else { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e5d3793bd4d204..75d1d9f6b2a794 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -79,6 +79,7 @@ increment_mutations(PyObject* dict) { * so we don't need to check that they haven't been used */ #define BUILTINS_WATCHER_ID 0 #define GLOBALS_WATCHER_ID 1 +#define TYPE_WATCHER_ID 0 static int globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, @@ -92,6 +93,14 @@ globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, return 0; } +static int +type_watcher_callback(PyTypeObject* type) +{ + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), type, 1); + PyType_Unwatch(TYPE_WATCHER_ID, (PyObject *)type); + return 0; +} + static PyObject * convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) { @@ -167,6 +176,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; } + if (interp->type_watchers[TYPE_WATCHER_ID] == NULL) { + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + } for (int pc = 0; pc < buffer_size; pc++) { _PyUOpInstruction *inst = &buffer[pc]; int opcode = inst->opcode; @@ -310,9 +322,11 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_has_type _Py_uop_sym_has_type #define sym_get_type _Py_uop_sym_get_type #define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version #define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) #define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) #define sym_is_bottom _Py_uop_sym_is_bottom #define sym_truthiness _Py_uop_sym_truthiness @@ -395,7 +409,7 @@ optimize_uops( _PyUOpInstruction *corresponding_check_stack = NULL; _Py_uop_abstractcontext_init(ctx); - _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); if (frame == NULL) { return -1; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index a2cb4c0b2c5192..e6fb85a90603eb 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -21,11 +21,13 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_new_const _Py_uop_sym_new_const #define sym_new_null _Py_uop_sym_new_null #define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version #define sym_get_type _Py_uop_sym_get_type #define sym_has_type _Py_uop_sym_has_type #define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) #define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) #define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new @@ -113,6 +115,29 @@ dummy_func(void) { sym_set_type(right, &PyLong_Type); } + op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + + } + } + op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { if (sym_matches_type(left, &PyFloat_Type)) { if (sym_matches_type(right, &PyFloat_Type)) { @@ -563,16 +588,12 @@ dummy_func(void) { argcount++; } - _Py_UopsSymbol **localsplus_start = ctx->n_consumed; - int n_locals_already_filled = 0; - // Can determine statically, so we interleave the new locals - // and make the current stack the new locals. - // This also sets up for true call inlining. if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - localsplus_start = args; - n_locals_already_filled = argcount; + new_frame = frame_new(ctx, co, 0, args, argcount); + } else { + new_frame = frame_new(ctx, co, 0, NULL, 0); + } - new_frame = frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0); } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b3787345ec6714..18f3ca4cb73e5a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -930,6 +930,28 @@ } case _GUARD_TYPE_VERSION: { + _Py_UopsSymbol *owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)this_instr->operand; + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + } break; } @@ -1583,16 +1605,11 @@ args--; argcount++; } - _Py_UopsSymbol **localsplus_start = ctx->n_consumed; - int n_locals_already_filled = 0; - // Can determine statically, so we interleave the new locals - // and make the current stack the new locals. - // This also sets up for true call inlining. if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - localsplus_start = args; - n_locals_already_filled = argcount; + new_frame = frame_new(ctx, co, 0, args, argcount); + } else { + new_frame = frame_new(ctx, co, 0, NULL, 0); } - new_frame = frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0); stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1 - oparg; break; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index e546eef306eeca..f3d4078bf1a890 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -52,7 +52,8 @@ static inline int get_lltrace(void) { static _Py_UopsSymbol NO_SPACE_SYMBOL = { .flags = IS_NULL | NOT_NULL | NO_SPACE, .typ = NULL, - .const_val = NULL + .const_val = NULL, + .type_version = 0, }; _Py_UopsSymbol * @@ -76,6 +77,7 @@ sym_new(_Py_UOpsContext *ctx) self->flags = 0; self->typ = NULL; self->const_val = NULL; + self->type_version = 0; return self; } @@ -152,6 +154,18 @@ _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *ty } } +bool +_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version) +{ + // if the type version was already set, then it must be different and we should set it to bottom + if (sym->type_version) { + sym_set_bottom(ctx, sym); + return false; + } + sym->type_version = version; + return true; +} + void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val) { @@ -256,6 +270,12 @@ _Py_uop_sym_get_type(_Py_UopsSymbol *sym) return sym->typ; } +unsigned int +_Py_uop_sym_get_type_version(_Py_UopsSymbol *sym) +{ + return sym->type_version; +} + bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym) { @@ -272,6 +292,13 @@ _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) return _Py_uop_sym_get_type(sym) == typ; } +bool +_Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version) +{ + return _Py_uop_sym_get_type_version(sym) == version; +} + + int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym) { @@ -311,9 +338,9 @@ _Py_UOpsAbstractFrame * _Py_uop_frame_new( _Py_UOpsContext *ctx, PyCodeObject *co, - _Py_UopsSymbol **localsplus_start, - int n_locals_already_filled, - int curr_stackentries) + int curr_stackentries, + _Py_UopsSymbol **args, + int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; @@ -321,19 +348,22 @@ _Py_uop_frame_new( frame->stack_len = co->co_stacksize; frame->locals_len = co->co_nlocalsplus; - frame->locals = localsplus_start; + frame->locals = ctx->n_consumed; frame->stack = frame->locals + co->co_nlocalsplus; frame->stack_pointer = frame->stack + curr_stackentries; - ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize); + ctx->n_consumed = ctx->n_consumed + (co->co_nlocalsplus + co->co_stacksize); if (ctx->n_consumed >= ctx->limit) { ctx->done = true; ctx->out_of_space = true; return NULL; } - // Initialize with the initial state of all local variables - for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) { + for (int i = 0; i < arg_len; i++) { + frame->locals[i] = args[i]; + } + + for (int i = arg_len; i < co->co_nlocalsplus; i++) { _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); frame->locals[i] = local; } From 38a25e9560cf0ff0b80d9e90bce793ff24c6e027 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sat, 8 Jun 2024 19:22:07 +0900 Subject: [PATCH 423/903] gh-120244: Fix re.sub() reference leak (GH-120245) --- .../next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst | 1 + Modules/_sre/sre.c | 1 + 2 files changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst diff --git a/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst b/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst new file mode 100644 index 00000000000000..d21532f22a1d38 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst @@ -0,0 +1 @@ +Fix memory leak in :func:`re.sub()` when the replacement string contains backreferences. diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index c1eff63d921de9..e33034086481c2 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -1622,6 +1622,7 @@ _sre_template_impl(PyObject *module, PyObject *pattern, PyObject *template) } self->items[i].literal = Py_XNewRef(literal); } + PyObject_GC_Track(self); return (PyObject*) self; bad_template: From 5d59b870effa0f576acf7264cfcbfca2b36e34e3 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 9 Jun 2024 00:11:19 +0800 Subject: [PATCH 424/903] gh-120121: Add InvalidStateError to concurrent.futures.__all__ (#120123) Co-authored-by: Nikita Sobolev --- Lib/concurrent/futures/__init__.py | 1 + .../next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst | 1 + 2 files changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index 292e886d5a88ac..72de617a5b6f61 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -23,6 +23,7 @@ 'ALL_COMPLETED', 'CancelledError', 'TimeoutError', + 'InvalidStateError', 'BrokenExecutor', 'Future', 'Executor', diff --git a/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst b/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst new file mode 100644 index 00000000000000..4f3526477c8cce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst @@ -0,0 +1 @@ +Add :exc:`concurrent.futures.InvalidStateError` to module's ``__all__``. From 7c016deae62308dd1b4e2767fc6abf04857c7843 Mon Sep 17 00:00:00 2001 From: Clinton Date: Sat, 8 Jun 2024 13:18:58 -0400 Subject: [PATCH 425/903] gh-120276: Fix incorrect email.header.Header maxlinelen default (GH-120277) --- Doc/library/email.header.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index 6e230d5faf1654..219fad0d2f6745 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -77,7 +77,7 @@ Here is the :class:`Header` class description: The maximum line length can be specified explicitly via *maxlinelen*. For splitting the first line to a shorter value (to account for the field header which isn't included in *s*, e.g. :mailheader:`Subject`) pass in the name of the - field in *header_name*. The default *maxlinelen* is 76, and the default value + field in *header_name*. The default *maxlinelen* is 78, and the default value for *header_name* is ``None``, meaning it is not taken into account for the first line of a long, split header. From 34f5ae69fe9ab0f5b23311d5c396d0cbb5902913 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 8 Jun 2024 23:45:57 +0300 Subject: [PATCH 426/903] gh-120268: Prohibit passing ``None`` to ``_pydatetime.date.fromtimestamp`` (#120269) This makes the pure Python implementation consistent with the C implementation. --- Lib/_pydatetime.py | 2 ++ Lib/test/datetimetester.py | 5 +++++ .../Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst | 2 ++ 3 files changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index b7d569cc41740e..34ccb2da13d0f3 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -966,6 +966,8 @@ def __new__(cls, year, month=None, day=None): @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." + if t is None: + raise TypeError("'NoneType' object cannot be interpreted as an integer") y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b80da5697ef865..28f75a803b4e04 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1336,6 +1336,11 @@ def test_insane_fromtimestamp(self): self.assertRaises(OverflowError, self.theclass.fromtimestamp, insane) + def test_fromtimestamp_with_none_arg(self): + # See gh-120268 for more details + with self.assertRaises(TypeError): + self.theclass.fromtimestamp(None) + def test_today(self): import time diff --git a/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst b/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst new file mode 100644 index 00000000000000..d48d43cd047f7a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst @@ -0,0 +1,2 @@ +Prohibit passing ``None`` to pure-Python :meth:`datetime.date.fromtimestamp` +to achieve consistency with C-extension implementation. From 0ae8579b85f9b0cd3f287082ad6e194bdb025d88 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Sun, 9 Jun 2024 22:23:30 -0400 Subject: [PATCH 427/903] gh-119666: fix multiple class-scope comprehensions referencing __class__ (#120295) --- Lib/test/test_listcomps.py | 25 +++++++++++++++++++ ...-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst | 1 + Python/symtable.c | 23 ++++++++--------- 3 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index ec2aac81682db8..58b076e9ea5d8a 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -168,6 +168,31 @@ def test_references___class__(self): """ self._check_in_scopes(code, raises=NameError) + def test_references___class___defined(self): + code = """ + __class__ = 2 + res = [__class__ for x in [1]] + """ + self._check_in_scopes( + code, outputs={"res": [2]}, scopes=["module", "function"]) + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_references___class___enclosing(self): + code = """ + __class__ = 2 + class C: + res = [__class__ for x in [1]] + res = C.res + """ + self._check_in_scopes(code, raises=NameError) + + def test_super_and_class_cell_in_sibling_comps(self): + code = """ + [super for _ in [1]] + [__class__ for _ in [1]] + """ + self._check_in_scopes(code, raises=NameError) + def test_inner_cell_shadows_outer(self): code = """ items = [(lambda: i) for i in range(5)] diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst new file mode 100644 index 00000000000000..09c1f553c48702 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst @@ -0,0 +1 @@ +Fix a compiler crash in the case where two comprehensions in class scope both reference ``__class__``. diff --git a/Python/symtable.c b/Python/symtable.c index 0ee8ca36cf8df0..7e452cdb13badf 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -781,22 +781,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (existing == NULL && PyErr_Occurred()) { return 0; } + // __class__ is never allowed to be free through a class scope (see + // drop_class_free) + if (scope == FREE && ste->ste_type == ClassBlock && + _PyUnicode_EqualToASCIIString(k, "__class__")) { + scope = GLOBAL_IMPLICIT; + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } + remove_dunder_class = 1; + } if (!existing) { // name does not exist in scope, copy from comprehension assert(scope != FREE || PySet_Contains(comp_free, k) == 1); - if (scope == FREE && ste->ste_type == ClassBlock && - _PyUnicode_EqualToASCIIString(k, "__class__")) { - // if __class__ is unbound in the enclosing class scope and free - // in the comprehension scope, it needs special handling; just - // letting it be marked as free in class scope will break due to - // drop_class_free - scope = GLOBAL_IMPLICIT; - only_flags &= ~DEF_FREE; - if (PySet_Discard(comp_free, k) < 0) { - return 0; - } - remove_dunder_class = 1; - } PyObject *v_flags = PyLong_FromLong(only_flags); if (v_flags == NULL) { return 0; From e5a7bc6f2eb9a3875063423caa67bb0ffcc3a6b8 Mon Sep 17 00:00:00 2001 From: Clinton Date: Mon, 10 Jun 2024 04:17:50 -0400 Subject: [PATCH 428/903] gh-120296: Fix format string of fcntl.ioctl() audit (#120301) --- Modules/fcntlmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 873bdf2ac0657a..0c06c03a6c403e 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -170,7 +170,7 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, Py_ssize_t len; char buf[IOCTL_BUFSZ+1]; /* argument plus NUL byte */ - if (PySys_Audit("fcntl.ioctl", "iIO", fd, code, + if (PySys_Audit("fcntl.ioctl", "ikO", fd, code, ob_arg ? ob_arg : Py_None) < 0) { return NULL; } From 4829522b8d3e1a28930f1cccfcc9635e035a0eb4 Mon Sep 17 00:00:00 2001 From: "E. M. Bray" Date: Mon, 10 Jun 2024 10:55:49 +0200 Subject: [PATCH 429/903] bpo-24766: doc= argument to subclasses of property not handled correctly (GH-2487) Co-authored-by: Serhiy Storchaka --- Lib/test/test_property.py | 34 +++++++++++++++++++ .../2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst | 1 + Objects/descrobject.c | 19 +++-------- 3 files changed, 39 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index 408e64f53142db..b7a2219b96149a 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -463,6 +463,40 @@ def getter3(self): self.assertEqual(p.__doc__, "user") self.assertEqual(p2.__doc__, "user") + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_prefer_explicit_doc(self): + # Issue 25757: subclasses of property lose docstring + self.assertEqual(property(doc="explicit doc").__doc__, "explicit doc") + self.assertEqual(PropertySub(doc="explicit doc").__doc__, "explicit doc") + + class Foo: + spam = PropertySub(doc="spam explicit doc") + + @spam.getter + def spam(self): + """ignored as doc already set""" + return 1 + + def _stuff_getter(self): + """ignored as doc set directly""" + stuff = PropertySub(doc="stuff doc argument", fget=_stuff_getter) + + #self.assertEqual(Foo.spam.__doc__, "spam explicit doc") + self.assertEqual(Foo.stuff.__doc__, "stuff doc argument") + + def test_property_no_doc_on_getter(self): + # If a property's getter has no __doc__ then the property's doc should + # be None; test that this is consistent with subclasses as well; see + # GH-2487 + class NoDoc: + @property + def __doc__(self): + raise AttributeError + + self.assertEqual(property(NoDoc()).__doc__, None) + self.assertEqual(PropertySub(NoDoc()).__doc__, None) + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_setter_copies_getter_docstring(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst b/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst new file mode 100644 index 00000000000000..93a8562efe6d6f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst @@ -0,0 +1 @@ +Fix handling of ``doc`` argument to subclasses of ``property``. diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 1b7e2fde3ceccd..4eccd1704eb95a 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1859,22 +1859,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, /* if no docstring given and the getter has one, use that one */ else if (fget != NULL) { int rc = PyObject_GetOptionalAttr(fget, &_Py_ID(__doc__), &prop_doc); - if (rc <= 0) { + if (rc < 0) { return rc; } - if (!Py_IS_TYPE(self, &PyProperty_Type) && - prop_doc != NULL && prop_doc != Py_None) { - // This oddity preserves the long existing behavior of surfacing - // an AttributeError when using a dict-less (__slots__) property - // subclass as a decorator on a getter method with a docstring. - // See PropertySubclassTest.test_slots_docstring_copy_exception. - int err = PyObject_SetAttr( - (PyObject *)self, &_Py_ID(__doc__), prop_doc); - if (err < 0) { - Py_DECREF(prop_doc); // release our new reference. - return -1; - } - } if (prop_doc == Py_None) { prop_doc = NULL; Py_DECREF(Py_None); @@ -1902,7 +1889,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, Py_DECREF(prop_doc); if (err < 0) { assert(PyErr_Occurred()); - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (!self->getter_doc && + PyErr_ExceptionMatches(PyExc_AttributeError)) + { PyErr_Clear(); // https://github.com/python/cpython/issues/98963#issuecomment-1574413319 // Python silently dropped this doc assignment through 3.11. From b90bd3e5bbc136f53b24ee791824acd6b17e0d42 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 10 Jun 2024 11:54:35 +0200 Subject: [PATCH 430/903] gh-120155: Fix Coverity issue in zoneinfo load_data() (#120232) Declare the 'rv' varaible at the top of the load_data() function to make sure that it's initialized before the first 'goto error' which uses 'rv' (return rv). Fix the Coverity issue: Error: UNINIT (CWE-457): Python-3.12.2/Modules/_zoneinfo.c:1233:5: skipped_decl: Jumping over declaration of ""rv"". Python-3.12.2/Modules/_zoneinfo.c:1284:5: uninit_use: Using uninitialized value ""rv"". 1282| } 1283| 1284|-> return rv; 1285| } 1286| --- Modules/_zoneinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 38c3f0c45d803f..902ece795b575b 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -944,6 +944,7 @@ ttinfo_eq(const _ttinfo *const tti0, const _ttinfo *const tti1) static int load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) { + int rv = 0; PyObject *data_tuple = NULL; long *utcoff = NULL; @@ -1220,7 +1221,6 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } } - int rv = 0; goto cleanup; error: // These resources only need to be freed if we have failed, if we succeed From c3b6dbff2c8886de1edade737febe85dd47ff4d0 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 10 Jun 2024 13:06:18 +0200 Subject: [PATCH 431/903] gh-115801: Only allow sequence of strings as input for difflib.unified_diff (GH-118333) --- Lib/difflib.py | 6 ++++ Lib/test/test_difflib.py | 30 +++++++++++++++---- ...-04-27-18-36-46.gh-issue-115801.SVeHSy.rst | 1 + 3 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst diff --git a/Lib/difflib.py b/Lib/difflib.py index 0443963b4fd697..7f595b6c72e641 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1264,6 +1264,12 @@ def _check_types(a, b, *args): if b and not isinstance(b[0], str): raise TypeError('lines to compare must be str, not %s (%r)' % (type(b[0]).__name__, b[0])) + if isinstance(a, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(a).__name__) + if isinstance(b, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(b).__name__) for arg in args: if not isinstance(arg, str): raise TypeError('all arguments must be str, not: %r' % (arg,)) diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index bf6e5b1152b4a2..9e217249be7332 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -295,7 +295,7 @@ def test_close_matches_aligned(self): class TestOutputFormat(unittest.TestCase): def test_tab_delimiter(self): - args = ['one', 'two', 'Original', 'Current', + args = [['one'], ['two'], 'Original', 'Current', '2005-01-26 23:30:50', '2010-04-02 10:20:52'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], [ @@ -307,7 +307,7 @@ def test_tab_delimiter(self): "--- Current\t2010-04-02 10:20:52"]) def test_no_trailing_tab_on_empty_filedate(self): - args = ['one', 'two', 'Original', 'Current'] + args = [['one'], ['two'], 'Original', 'Current'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], ["--- Original", "+++ Current"]) @@ -447,6 +447,28 @@ def assertDiff(expect, actual): lineterm=b'') assertDiff(expect, actual) + +class TestInputTypes(unittest.TestCase): + def _assert_type_error(self, msg, generator, *args): + with self.assertRaises(TypeError) as ctx: + list(generator(*args)) + self.assertEqual(msg, str(ctx.exception)) + + def test_input_type_checks(self): + unified = difflib.unified_diff + context = difflib.context_diff + + expect = "input must be a sequence of strings, not str" + self._assert_type_error(expect, unified, 'a', ['b']) + self._assert_type_error(expect, context, 'a', ['b']) + + self._assert_type_error(expect, unified, ['a'], 'b') + self._assert_type_error(expect, context, ['a'], 'b') + + expect = "lines to compare must be str, not NoneType (None)" + self._assert_type_error(expect, unified, ['a'], [None]) + self._assert_type_error(expect, context, ['a'], [None]) + def test_mixed_types_content(self): # type of input content must be consistent: all str or all bytes a = [b'hello'] @@ -495,10 +517,6 @@ def test_mixed_types_dates(self): b = ['bar\n'] list(difflib.unified_diff(a, b, 'a', 'b', datea, dateb)) - def _assert_type_error(self, msg, generator, *args): - with self.assertRaises(TypeError) as ctx: - list(generator(*args)) - self.assertEqual(msg, str(ctx.exception)) class TestJunkAPIs(unittest.TestCase): def test_is_line_junk_true(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst new file mode 100644 index 00000000000000..93b176d5767335 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst @@ -0,0 +1 @@ +Raise ``TypeError`` when passing a string to :func:`difflib.unified_diff` and :func:`difflib.context_diff`. From 56c3815ba14c790d2e9a227b4ac0ead5e6b1e570 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:15:12 +0100 Subject: [PATCH 432/903] gh-119786: copy compiler doc from devguide to InternalDocs and convert to markdown (#120134) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gh-119876: move compiler doc from devguide to InternalDocs Copy of https://github.com/python/devguide/commit/78fc0d7aa9fd0d6733d10c23b178b2a0e2799afc Co-Authored-By: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-Authored-By: Adam Turner <9087854+aa-turner@users.noreply.github.com> Co-Authored-By: Brett Cannon Co-Authored-By: Carol Willing Co-Authored-By: Daniel Porteous Co-Authored-By: Dennis Sweeney <36520290+sweeneyde@users.noreply.github.com> Co-Authored-By: Éric Araujo Co-Authored-By: Erlend Egeberg Aasland Co-Authored-By: Ezio Melotti Co-Authored-By: Georg Brandl Co-Authored-By: Guido van Rossum Co-Authored-By: Hugo van Kemenade Co-Authored-By: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Co-Authored-By: Jeff Allen Co-Authored-By: Jim Fasarakis-Hilliard Co-Authored-By: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Co-Authored-By: Lincoln <71312724+Lincoln-developer@users.noreply.github.com> Co-Authored-By: Mariatta Co-Authored-By: Muhammad Mahad Co-Authored-By: Ned Deily Co-Authored-By: Pablo Galindo Salgado Co-Authored-By: Serhiy Storchaka Co-Authored-By: Stéphane Wirtel Co-Authored-By: Suriyaa ✌️️ Co-Authored-By: Zachary Ware Co-Authored-By: psyker156 <242220+psyker156@users.noreply.github.com> Co-Authored-By: slateny <46876382+slateny@users.noreply.github.com> Co-Authored-By: svelankar <17737361+svelankar@users.noreply.github.com> Co-Authored-By: zikcheng * convert to markdown * add to index * update more of the out of date stuff --------- Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Brett Cannon Co-authored-by: Carol Willing Co-authored-by: Daniel Porteous Co-authored-by: Dennis Sweeney <36520290+sweeneyde@users.noreply.github.com> Co-authored-by: Éric Araujo Co-authored-by: Erlend Egeberg Aasland Co-authored-by: Ezio Melotti Co-authored-by: Georg Brandl Co-authored-by: Guido van Rossum Co-authored-by: Hugo van Kemenade Co-authored-by: Jeff Allen Co-authored-by: Jim Fasarakis-Hilliard Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Co-authored-by: Lincoln <71312724+Lincoln-developer@users.noreply.github.com> Co-authored-by: Mariatta Co-authored-by: Muhammad Mahad Co-authored-by: Ned Deily Co-authored-by: Pablo Galindo Salgado Co-authored-by: Serhiy Storchaka Co-authored-by: Stéphane Wirtel Co-authored-by: Suriyaa ✌️️ Co-authored-by: Zachary Ware Co-authored-by: psyker156 <242220+psyker156@users.noreply.github.com> Co-authored-by: slateny <46876382+slateny@users.noreply.github.com> Co-authored-by: svelankar <17737361+svelankar@users.noreply.github.com> Co-authored-by: zikcheng --- InternalDocs/README.md | 2 + InternalDocs/compiler.md | 651 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 653 insertions(+) create mode 100644 InternalDocs/compiler.md diff --git a/InternalDocs/README.md b/InternalDocs/README.md index a2502fbf198735..42f6125794266a 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -12,6 +12,8 @@ it is not, please report that through the [issue tracker](https://github.com/python/cpython/issues). +[Compiler Design](compiler.md) + [Exception Handling](exception_handling.md) [Adaptive Instruction Families](adaptive.md) diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md new file mode 100644 index 00000000000000..0abc10da6e05c6 --- /dev/null +++ b/InternalDocs/compiler.md @@ -0,0 +1,651 @@ + +Compiler design +=============== + +Abstract +-------- + +In CPython, the compilation from source code to bytecode involves several steps: + +1. Tokenize the source code + [Parser/lexer/](https://github.com/python/cpython/blob/main/Parser/lexer/) + and [Parser/tokenizer/](https://github.com/python/cpython/blob/main/Parser/tokenizer/). +2. Parse the stream of tokens into an Abstract Syntax Tree + [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c). +3. Transform AST into an instruction sequence + [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c). +4. Construct a Control Flow Graph and apply optimizations to it + [Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c). +5. Emit bytecode based on the Control Flow Graph + [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c). + +This document outlines how these steps of the process work. + +This document only describes parsing in enough depth to explain what is needed +for understanding compilation. This document provides a detailed, though not +exhaustive, view of the how the entire system works. You will most likely need +to read some source code to have an exact understanding of all details. + + +Parsing +======= + +As of Python 3.9, Python's parser is a PEG parser of a somewhat +unusual design. It is unusual in the sense that the parser's input is a stream +of tokens rather than a stream of characters which is more common with PEG +parsers. + +The grammar file for Python can be found in +[Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram). +The definitions for literal tokens (such as ``:``, numbers, etc.) can be found in +[Grammar/Tokens](https://github.com/python/cpython/blob/main/Grammar/Tokens). +Various C files, including +[Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c) +are generated from these. + +See Also: + +* [Guide to the parser](https://devguide.python.org/internals/parser/index.html) + for a detailed description of the parser. + +* [Changing CPython’s grammar](https://devguide.python.org/developer-workflow/grammar/#grammar) + for a detailed description of the grammar. + + +Abstract syntax trees (AST) +=========================== + + +The abstract syntax tree (AST) is a high-level representation of the +program structure without the necessity of containing the source code; +it can be thought of as an abstract representation of the source code. The +specification of the AST nodes is specified using the Zephyr Abstract +Syntax Definition Language (ASDL) [^1], [^2]. + +The definition of the AST nodes for Python is found in the file +[Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl). + +Each AST node (representing statements, expressions, and several +specialized types, like list comprehensions and exception handlers) is +defined by the ASDL. Most definitions in the AST correspond to a +particular source construct, such as an 'if' statement or an attribute +lookup. The definition is independent of its realization in any +particular programming language. + +The following fragment of the Python ASDL construct demonstrates the +approach and syntax: + +``` + module Python + { + stmt = FunctionDef(identifier name, arguments args, stmt* body, + expr* decorators) + | Return(expr? value) | Yield(expr? value) + attributes (int lineno) + } +``` + +The preceding example describes two different kinds of statements and an +expression: function definitions, return statements, and yield expressions. +All three kinds are considered of type ``stmt`` as shown by ``|`` separating +the various kinds. They all take arguments of various kinds and amounts. + +Modifiers on the argument type specify the number of values needed; ``?`` +means it is optional, ``*`` means 0 or more, while no modifier means only one +value for the argument and it is required. ``FunctionDef``, for instance, +takes an ``identifier`` for the *name*, ``arguments`` for *args*, zero or more +``stmt`` arguments for *body*, and zero or more ``expr`` arguments for +*decorators*. + +Do notice that something like 'arguments', which is a node type, is +represented as a single AST node and not as a sequence of nodes as with +stmt as one might expect. + +All three kinds also have an 'attributes' argument; this is shown by the +fact that 'attributes' lacks a '|' before it. + +The statement definitions above generate the following C structure type: + + +``` + typedef struct _stmt *stmt_ty; + + struct _stmt { + enum { FunctionDef_kind=1, Return_kind=2, Yield_kind=3 } kind; + union { + struct { + identifier name; + arguments_ty args; + asdl_seq *body; + } FunctionDef; + + struct { + expr_ty value; + } Return; + + struct { + expr_ty value; + } Yield; + } v; + int lineno; + } +``` + +Also generated are a series of constructor functions that allocate (in +this case) a ``stmt_ty`` struct with the appropriate initialization. The +``kind`` field specifies which component of the union is initialized. The +``FunctionDef()`` constructor function sets 'kind' to ``FunctionDef_kind`` and +initializes the *name*, *args*, *body*, and *attributes* fields. + +See also +[Green Tree Snakes - The missing Python AST docs](https://greentreesnakes.readthedocs.io/en/latest) + by Thomas Kluyver. + +Memory management +================= + +Before discussing the actual implementation of the compiler, a discussion of +how memory is handled is in order. To make memory management simple, an **arena** +is used that pools memory in a single location for easy +allocation and removal. This enables the removal of explicit memory +deallocation. Because memory allocation for all needed memory in the compiler +registers that memory with the arena, a single call to free the arena is all +that is needed to completely free all memory used by the compiler. + +In general, unless you are working on the critical core of the compiler, memory +management can be completely ignored. But if you are working at either the +very beginning of the compiler or the end, you need to care about how the arena +works. All code relating to the arena is in either +[Include/internal/pycore_pyarena.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_pyarena.h) +or [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c). + +``PyArena_New()`` will create a new arena. The returned ``PyArena`` structure +will store pointers to all memory given to it. This does the bookkeeping of +what memory needs to be freed when the compiler is finished with the memory it +used. That freeing is done with ``PyArena_Free()``. This only needs to be +called in strategic areas where the compiler exits. + +As stated above, in general you should not have to worry about memory +management when working on the compiler. The technical details of memory +management have been designed to be hidden from you for most cases. + +The only exception comes about when managing a PyObject. Since the rest +of Python uses reference counting, there is extra support added +to the arena to cleanup each PyObject that was allocated. These cases +are very rare. However, if you've allocated a PyObject, you must tell +the arena about it by calling ``PyArena_AddPyObject()``. + + +Source code to AST +================== + +The AST is generated from source code using the function +``_PyParser_ASTFromString()`` or ``_PyParser_ASTFromFile()`` +[Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c). + +After some checks, a helper function in +[Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c) +begins applying production rules on the source code it receives; converting source +code to tokens and matching these tokens recursively to their corresponding rule. The +production rule's corresponding rule function is called on every match. These rule +functions follow the format `xx_rule`. Where *xx* is the grammar rule +that the function handles and is automatically derived from +[Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram) by +[Tools/peg_generator/pegen/c_generator.py](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen/c_generator.py). + +Each rule function in turn creates an AST node as it goes along. It does this +by allocating all the new nodes it needs, calling the proper AST node creation +functions for any required supporting functions and connecting them as needed. +This continues until all nonterminal symbols are replaced with terminals. If an +error occurs, the rule functions backtrack and try another rule function. If +there are no more rules, an error is set and the parsing ends. + +The AST node creation helper functions have the name `_PyAST_{xx}` +where *xx* is the AST node that the function creates. These are defined by the +ASDL grammar and contained in +[Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) +(which is generated by +[Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py) +from +[Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl)). +This all leads to a sequence of AST nodes stored in ``asdl_seq`` structs. + +To demonstrate everything explained so far, here's the +rule function responsible for a simple named import statement such as +``import sys``. Note that error-checking and debugging code has been +omitted. Removed parts are represented by ``...``. +Furthermore, some comments have been added for explanation. These comments +may not be present in the actual code. + + +``` + // This is the production rule (from python.gram) the rule function + // corresponds to: + // import_name: 'import' dotted_as_names + static stmt_ty + import_name_rule(Parser *p) + { + ... + stmt_ty _res = NULL; + { // 'import' dotted_as_names + ... + Token * _keyword; + asdl_alias_seq* a; + // The tokenizing steps. + if ( + (_keyword = _PyPegen_expect_token(p, 513)) // token='import' + && + (a = dotted_as_names_rule(p)) // dotted_as_names + ) + { + ... + // Generate an AST for the import statement. + _res = _PyAST_Import ( a , ...); + ... + goto done; + } + ... + } + _res = NULL; + done: + ... + return _res; + } +``` + + +To improve backtracking performance, some rules (chosen by applying a +``(memo)`` flag in the grammar file) are memoized. Each rule function checks if +a memoized version exists and returns that if so, else it continues in the +manner stated in the previous paragraphs. + +There are macros for creating and using ``asdl_xx_seq *`` types, where *xx* is +a type of the ASDL sequence. Three main types are defined +manually -- ``generic``, ``identifier`` and ``int``. These types are found in +[Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c) +and its corresponding header file +[Include/internal/pycore_asdl.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_asdl.h). +Functions and macros for creating ``asdl_xx_seq *`` types are as follows: + +``_Py_asdl_generic_seq_new(Py_ssize_t, PyArena *)`` + Allocate memory for an ``asdl_generic_seq`` of the specified length +``_Py_asdl_identifier_seq_new(Py_ssize_t, PyArena *)`` + Allocate memory for an ``asdl_identifier_seq`` of the specified length +``_Py_asdl_int_seq_new(Py_ssize_t, PyArena *)`` + Allocate memory for an ``asdl_int_seq`` of the specified length + +In addition to the three types mentioned above, some ASDL sequence types are +automatically generated by +[Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py) +and found in +[Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h). +Macros for using both manually defined and automatically generated ASDL +sequence types are as follows: + +``asdl_seq_GET(asdl_xx_seq *, int)`` + Get item held at a specific position in an ``asdl_xx_seq`` +``asdl_seq_SET(asdl_xx_seq *, int, stmt_ty)`` + Set a specific index in an ``asdl_xx_seq`` to the specified value + +Untyped counterparts exist for some of the typed macros. These are useful +when a function needs to manipulate a generic ASDL sequence: + +``asdl_seq_GET_UNTYPED(asdl_seq *, int)`` + Get item held at a specific position in an ``asdl_seq`` +``asdl_seq_SET_UNTYPED(asdl_seq *, int, stmt_ty)`` + Set a specific index in an ``asdl_seq`` to the specified value +``asdl_seq_LEN(asdl_seq *)`` + Return the length of an ``asdl_seq`` or ``asdl_xx_seq`` + +Note that typed macros and functions are recommended over their untyped +counterparts. Typed macros carry out checks in debug mode and aid +debugging errors caused by incorrectly casting from ``void *``. + +If you are working with statements, you must also worry about keeping +track of what line number generated the statement. Currently the line +number is passed as the last parameter to each ``stmt_ty`` function. + +See also [PEP 617: New PEG parser for CPython](https://peps.python.org/pep-0617/). + + +Control flow graphs +=================== + +A **control flow graph** (often referenced by its acronym, **CFG**) is a +directed graph that models the flow of a program. A node of a CFG is +not an individual bytecode instruction, but instead represents a +sequence of bytecode instructions that always execute sequentially. +Each node is called a *basic block* and must always execute from +start to finish, with a single entry point at the beginning and a +single exit point at the end. If some bytecode instruction *a* needs +to jump to some other bytecode instruction *b*, then *a* must occur at +the end of its basic block, and *b* must occur at the start of its +basic block. + +As an example, consider the following code snippet: + +.. code-block:: Python + + if x < 10: + f1() + f2() + else: + g() + end() + +The ``x < 10`` guard is represented by its own basic block that +compares ``x`` with ``10`` and then ends in a conditional jump based on +the result of the comparison. This conditional jump allows the block +to point to both the body of the ``if`` and the body of the ``else``. The +``if`` basic block contains the ``f1()`` and ``f2()`` calls and points to +the ``end()`` basic block. The ``else`` basic block contains the ``g()`` +call and similarly points to the ``end()`` block. + +Note that more complex code in the guard, the ``if`` body, or the ``else`` +body may be represented by multiple basic blocks. For instance, +short-circuiting boolean logic in a guard like ``if x or y:`` +will produce one basic block that tests the truth value of ``x`` +and then points both (1) to the start of the ``if`` body and (2) to +a different basic block that tests the truth value of y. + +CFGs are useful as an intermediate representation of the code because +they are a convenient data structure for optimizations. + +AST to CFG to bytecode +====================== + +The conversion of an ``AST`` to bytecode is initiated by a call to the function +``_PyAST_Compile()`` in +[Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c). + +The first step is to construct the symbol table. This is implemented by +``_PySymtable_Build()`` in +[Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c). +This function begins by entering the starting code block for the AST (passed-in) +and then calling the proper `symtable_visit_{xx}` function (with *xx* being the +AST node type). Next, the AST tree is walked with the various code blocks that +delineate the reach of a local variable as blocks are entered and exited using +``symtable_enter_block()`` and ``symtable_exit_block()``, respectively. + +Once the symbol table is created, the ``AST`` is transformed by ``compiler_codegen()`` +in [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c) +into a sequence of pseudo instructions. These are similar to bytecode, but +in some cases they are more abstract, and are resolved later into actual +bytecode. The construction of this instruction sequence is handled by several +functions that break the task down by various AST node types. The functions are +all named `compiler_visit_{xx}` where *xx* is the name of the node type (such +as ``stmt``, ``expr``, etc.). Each function receives a ``struct compiler *`` +and `{xx}_ty` where *xx* is the AST node type. Typically these functions +consist of a large 'switch' statement, branching based on the kind of +node type passed to it. Simple things are handled inline in the +'switch' statement with more complex transformations farmed out to other +functions named `compiler_{xx}` with *xx* being a descriptive name of what is +being handled. + +When transforming an arbitrary AST node, use the ``VISIT()`` macro. +The appropriate `compiler_visit_{xx}` function is called, based on the value +passed in for (so `VISIT({c}, expr, {node})` calls +`compiler_visit_expr({c}, {node})`). The ``VISIT_SEQ()`` macro is very similar, +but is called on AST node sequences (those values that were created as +arguments to a node that used the '*' modifier). + +Emission of bytecode is handled by the following macros: + +* ``ADDOP(struct compiler *, location, int)`` + add a specified opcode +* ``ADDOP_IN_SCOPE(struct compiler *, location, int)`` + like ``ADDOP``, but also exits current scope; used for adding return value + opcodes in lambdas and closures +* ``ADDOP_I(struct compiler *, location, int, Py_ssize_t)`` + add an opcode that takes an integer argument +* ``ADDOP_O(struct compiler *, location, int, PyObject *, TYPE)`` + add an opcode with the proper argument based on the position of the + specified PyObject in PyObject sequence object, but with no handling of + mangled names; used for when you + need to do named lookups of objects such as globals, consts, or + parameters where name mangling is not possible and the scope of the + name is known; *TYPE* is the name of PyObject sequence + (``names`` or ``varnames``) +* ``ADDOP_N(struct compiler *, location, int, PyObject *, TYPE)`` + just like ``ADDOP_O``, but steals a reference to PyObject +* ``ADDOP_NAME(struct compiler *, location, int, PyObject *, TYPE)`` + just like ``ADDOP_O``, but name mangling is also handled; used for + attribute loading or importing based on name +* ``ADDOP_LOAD_CONST(struct compiler *, location, PyObject *)`` + add the ``LOAD_CONST`` opcode with the proper argument based on the + position of the specified PyObject in the consts table. +* ``ADDOP_LOAD_CONST_NEW(struct compiler *, location, PyObject *)`` + just like ``ADDOP_LOAD_CONST_NEW``, but steals a reference to PyObject +* ``ADDOP_JUMP(struct compiler *, location, int, basicblock *)`` + create a jump to a basic block + +The ``location`` argument is a struct with the source location to be +associated with this instruction. It is typically extracted from an +``AST`` node with the ``LOC`` macro. The ``NO_LOCATION`` can be used +for *synthetic* instructions, which we do not associate with a line +number at this stage. For example, the implicit ``return None`` +which is added at the end of a function is not associated with any +line in the source code. + +There are several helper functions that will emit pseudo-instructions +and are named `compiler_{xx}()` where *xx* is what the function helps +with (``list``, ``boolop``, etc.). A rather useful one is ``compiler_nameop()``. +This function looks up the scope of a variable and, based on the +expression context, emits the proper opcode to load, store, or delete +the variable. + +Once the instruction sequence is created, it is transformed into a CFG +by ``_PyCfg_FromInstructionSequence()``. Then ``_PyCfg_OptimizeCodeUnit()`` +applies various peephole optimizations, and +``_PyCfg_OptimizedCfgToInstructionSequence()`` converts the optimized ``CFG`` +back into an instruction sequence. These conversions and optimizations are +implemented in +[Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c). + +Finally, the sequence of pseudo-instructions is converted into actual +bytecode. This includes transforming pseudo instructions into actual instructions, +converting jump targets from logical labels to relative offsets, and +construction of the +[exception table](exception_handling.md) and +[locations table](https://github.com/python/cpython/blob/main/Objects/locations.md). +The bytecode and tables are then wrapped into a ``PyCodeObject`` along with additional +metadata, including the ``consts`` and ``names`` arrays, information about function +reference to the source code (filename, etc). All of this is implemented by +``_PyAssemble_MakeCodeObject()`` in +[Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c). + + +Code objects +============ + +The result of ``PyAST_CompileObject()`` is a ``PyCodeObject`` which is defined in +[Include/cpython/code.h](https://github.com/python/cpython/blob/main/Include/cpython/code.h). +And with that you now have executable Python bytecode! + +The code objects (byte code) are executed in +[Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c). +This file will also need a new case statement for the new opcode in the big switch +statement in ``_PyEval_EvalFrameDefault()``. + + +Important files +=============== + +* [Parser/](https://github.com/python/cpython/blob/main/Parser/) + + * [Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl): + ASDL syntax file. + + * [Parser/asdl.py](https://github.com/python/cpython/blob/main/Parser/asdl.py): + Parser for ASDL definition files. + Reads in an ASDL description and parses it into an AST that describes it. + + * [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py): + Generate C code from an ASDL description. Generates + [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) + and + [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h). + + * [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c): + The new PEG parser introduced in Python 3.9. + Generated by + [Tools/peg_generator/pegen/c_generator.py](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen/c_generator.py) + from the grammar [Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram). + Creates the AST from source code. Rule functions for their corresponding production + rules are found here. + + * [Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c): + Contains high-level functions which are + used by the interpreter to create an AST from source code. + + * [Parser/pegen.c](https://github.com/python/cpython/blob/main/Parser/pegen.c): + Contains helper functions which are used by functions in + [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c) + to construct the AST. Also contains helper functions which help raise better error messages + when parsing source code. + + * [Parser/pegen.h](https://github.com/python/cpython/blob/main/Parser/pegen.h): + Header file for the corresponding + [Parser/pegen.c](https://github.com/python/cpython/blob/main/Parser/pegen.c). + Also contains definitions of the ``Parser`` and ``Token`` structs. + +* [Python/](https://github.com/python/cpython/blob/main/Python) + + * [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c): + Creates C structs corresponding to the ASDL types. Also contains code for + marshalling AST nodes (core ASDL types have marshalling code in + [Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c)). + "File automatically generated by + [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py). + This file must be committed separately after every grammar change + is committed since the ``__version__`` value is set to the latest + grammar change revision number. + + * [Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c): + Contains code to handle the ASDL sequence type. + Also has code to handle marshalling the core ASDL types, such as number + and identifier. Used by + [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) + for marshalling AST nodes. + + * [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c): + Used for validating the AST. + + * [Python/ast_opt.c](https://github.com/python/cpython/blob/main/Python/ast_opt.c): + Optimizes the AST. + + * [Python/ast_unparse.c](https://github.com/python/cpython/blob/main/Python/ast_unparse.c): + Converts the AST expression node back into a string (for string annotations). + + * [Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c): + Executes byte code (aka, eval loop). + + * [Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c): + Generates a symbol table from AST. + + * [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c): + Implementation of the arena memory manager. + + * [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c): + Emits pseudo bytecode based on the AST. + + * [Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c): + Implements peephole optimizations. + + * [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c): + Constructs a code object from a sequence of pseudo instructions. + + * [Python/instruction_sequence.c.c](https://github.com/python/cpython/blob/main/Python/instruction_sequence.c.c): + A data structure representing a sequence of bytecode-like pseudo-instructions. + +* [Include/](https://github.com/python/cpython/blob/main/Include/) + + * [Include/cpython/code.h](https://github.com/python/cpython/blob/main/Include/cpython/code.h) + : Header file for + [Objects/codeobject.c](https://github.com/python/cpython/blob/main/Objects/codeobject.c); + contains definition of ``PyCodeObject``. + + * [Include/opcode.h](https://github.com/python/cpython/blob/main/Include/opcode.h) + : One of the files that must be modified if + [Lib/opcode.py](https://github.com/python/cpython/blob/main/Lib/opcode.py) is. + + * [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h) + : Contains the actual definitions of the C structs as generated by + [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) + "Automatically generated by + [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py). + + * [Include/internal/pycore_asdl.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_asdl.h) + : Header for the corresponding + [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c). + + * [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h) + : Declares ``_PyAST_Validate()`` external (from + [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c)). + + * [Include/internal/pycore_symtable.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_symtable.h) + : Header for + [Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c). + ``struct symtable`` and ``PySTEntryObject`` are defined here. + + * [Include/internal/pycore_parser.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_parser.h) + : Header for the corresponding + [Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c). + + * [Include/internal/pycore_pyarena.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_pyarena.h) + : Header file for the corresponding + [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c). + + * [Include/opcode_ids.h](https://github.com/python/cpython/blob/main/Include/opcode_ids.h) + : List of opcodes. Generated from + [Python/bytecodes.c](https://github.com/python/cpython/blob/main/Python/bytecodes.c) + by + [Tools/cases_generator/opcode_id_generator.py](https://github.com/python/cpython/blob/main/Tools/cases_generator/opcode_id_generator.py). + +* [Objects/](https://github.com/python/cpython/blob/main/Objects/) + + * [Objects/codeobject.c](https://github.com/python/cpython/blob/main/Objects/codeobject.c) + : Contains PyCodeObject-related code. + + * [Objects/frameobject.c](https://github.com/python/cpython/blob/main/Objects/frameobject.c) + : Contains the ``frame_setlineno()`` function which should determine whether it is allowed + to make a jump between two points in a bytecode. + +* [Lib/](https://github.com/python/cpython/blob/main/Lib/) + + * [Lib/opcode.py](https://github.com/python/cpython/blob/main/Lib/opcode.py) + : opcode utilities exposed to Python. + + * [Lib/importlib/_bootstrap_external.py](https://github.com/python/cpython/blob/main/Lib/importlib/_bootstrap_external.py) + : Home of the magic number (named ``MAGIC_NUMBER``) for bytecode versioning. + + +Objects +======= + +* [Objects/locations.md](https://github.com/python/cpython/blob/main/Objects/locations.md): Describes the location table +* [Objects/frame_layout.md](https://github.com/python/cpython/blob/main/Objects/frame_layout.md): Describes the frame stack +* [Objects/object_layout.md](https://github.com/python/cpython/blob/main/Objects/object_layout.md): Descibes object layout for 3.11 and later +* [Exception Handling](exception_handling.md): Describes the exception table + + +Specializing Adaptive Interpreter +================================= + +Adding a specializing, adaptive interpreter to CPython will bring significant +performance improvements. These documents provide more information: + +* [PEP 659: Specializing Adaptive Interpreter](https://peps.python.org/pep-0659/). +* [Adding or extending a family of adaptive instructions](adaptive.md) + + +References +========== + +[^1]: Daniel C. Wang, Andrew W. Appel, Jeff L. Korn, and Chris + S. Serra. `The Zephyr Abstract Syntax Description Language.`_ + In Proceedings of the Conference on Domain-Specific Languages, + pp. 213--227, 1997. + +[^2]: The Zephyr Abstract Syntax Description Language.: + https://www.cs.princeton.edu/research/techreps/TR-554-97 From 7aff2de62bc28eb23888270b698c6b6915f69b21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 10 Jun 2024 18:34:17 +0200 Subject: [PATCH 433/903] gh-120057: Add os.environ.refresh() method (#120059) --- Doc/library/os.rst | 11 +++++ Doc/whatsnew/3.14.rst | 7 +++ Lib/os.py | 25 ++++++++-- Lib/test/test_os.py | 46 +++++++++++++++++++ ...-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst | 4 ++ Modules/clinic/posixmodule.c.h | 20 +++++++- Modules/posixmodule.c | 15 ++++++ 7 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index b93b06d4e72afc..360d71e70960c7 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -193,6 +193,10 @@ process and user. to the environment made after this time are not reflected in :data:`os.environ`, except for changes made by modifying :data:`os.environ` directly. + The :meth:`!os.environ.refresh()` method updates :data:`os.environ` with + changes to the environment made by :func:`os.putenv`, by + :func:`os.unsetenv`, or made outside Python in the same process. + This mapping may be used to modify the environment as well as query the environment. :func:`putenv` will be called automatically when the mapping is modified. @@ -225,6 +229,9 @@ process and user. .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. + .. versionchanged:: 3.14 + Added the :meth:`!os.environ.refresh()` method. + .. data:: environb @@ -561,6 +568,8 @@ process and user. of :data:`os.environ`. This also applies to :func:`getenv` and :func:`getenvb`, which respectively use :data:`os.environ` and :data:`os.environb` in their implementations. + See also the :data:`os.environ.refresh() ` method. + .. note:: On some platforms, including FreeBSD and macOS, setting ``environ`` may @@ -809,6 +818,8 @@ process and user. don't update :data:`os.environ`, so it is actually preferable to delete items of :data:`os.environ`. + See also the :data:`os.environ.refresh() ` method. + .. audit-event:: os.unsetenv key os.unsetenv .. versionchanged:: 3.9 diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b2dd80b64a691a..b77ff30a8fbbee 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -92,6 +92,13 @@ ast Added :func:`ast.compare` for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.) +os +-- + +* Added the :data:`os.environ.refresh() ` method to update + :data:`os.environ` with changes to the environment made by :func:`os.putenv`, + by :func:`os.unsetenv`, or made outside Python in the same process. + (Contributed by Victor Stinner in :gh:`120057`.) Optimizations diff --git a/Lib/os.py b/Lib/os.py index 0408e2db79e66e..4b48afb040e565 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -64,6 +64,10 @@ def _get_exports_list(module): from posix import _have_functions except ImportError: pass + try: + from posix import _create_environ + except ImportError: + pass import posix __all__.extend(_get_exports_list(posix)) @@ -88,6 +92,10 @@ def _get_exports_list(module): from nt import _have_functions except ImportError: pass + try: + from nt import _create_environ + except ImportError: + pass else: raise ImportError('no os specific module found') @@ -773,7 +781,18 @@ def __ror__(self, other): new.update(self) return new -def _createenviron(): + if _exists("_create_environ"): + def refresh(self): + data = _create_environ() + if name == 'nt': + data = {self.encodekey(key): value + for key, value in data.items()} + + # modify in-place to keep os.environb in sync + self._data.clear() + self._data.update(data) + +def _create_environ_mapping(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE def check_str(value): @@ -803,8 +822,8 @@ def decode(value): encode, decode) # unicode environ -environ = _createenviron() -del _createenviron +environ = _create_environ_mapping() +del _create_environ_mapping def getenv(key, default=None): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 2beb9ca8aa6ccb..f93937fb587386 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1298,6 +1298,52 @@ def test_ror_operator(self): self._test_underlying_process_env('_A_', '') self._test_underlying_process_env(overridden_key, original_value) + def test_refresh(self): + # Test os.environ.refresh() + has_environb = hasattr(os, 'environb') + + # Test with putenv() which doesn't update os.environ + os.environ['test_env'] = 'python_value' + os.putenv("test_env", "new_value") + self.assertEqual(os.environ['test_env'], 'python_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'python_value') + + os.environ.refresh() + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + # Test with unsetenv() which doesn't update os.environ + os.unsetenv('test_env') + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + os.environ.refresh() + self.assertNotIn('test_env', os.environ) + if has_environb: + self.assertNotIn(b'test_env', os.environb) + + if has_environb: + # test os.environb.refresh() with putenv() + os.environb[b'test_env'] = b'python_value2' + os.putenv("test_env", "new_value2") + self.assertEqual(os.environb[b'test_env'], b'python_value2') + self.assertEqual(os.environ['test_env'], 'python_value2') + + os.environb.refresh() + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + # test os.environb.refresh() with unsetenv() + os.unsetenv('test_env') + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + os.environb.refresh() + self.assertNotIn(b'test_env', os.environb) + self.assertNotIn('test_env', os.environ) class WalkTests(unittest.TestCase): """Tests for os.walk().""" diff --git a/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst b/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst new file mode 100644 index 00000000000000..955be59821ee0c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst @@ -0,0 +1,4 @@ +Added the :data:`os.environ.refresh() ` method to update +:data:`os.environ` with changes to the environment made by :func:`os.putenv`, +by :func:`os.unsetenv`, or made outside Python in the same process. +Patch by Victor Stinner. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 69fc178331c09c..07b28fef3a57ea 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -12152,6 +12152,24 @@ os__is_inputhook_installed(PyObject *module, PyObject *Py_UNUSED(ignored)) return os__is_inputhook_installed_impl(module); } +PyDoc_STRVAR(os__create_environ__doc__, +"_create_environ($module, /)\n" +"--\n" +"\n" +"Create the environment dictionary."); + +#define OS__CREATE_ENVIRON_METHODDEF \ + {"_create_environ", (PyCFunction)os__create_environ, METH_NOARGS, os__create_environ__doc__}, + +static PyObject * +os__create_environ_impl(PyObject *module); + +static PyObject * +os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__create_environ_impl(module); +} + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -12819,4 +12837,4 @@ os__is_inputhook_installed(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=faaa5e5ffb7b165d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5ae2e5ffcd9c8a84 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5f943d4b1c8085..a8fd5c494769b5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16809,6 +16809,20 @@ os__is_inputhook_installed_impl(PyObject *module) return PyBool_FromLong(PyOS_InputHook != NULL); } +/*[clinic input] +os._create_environ + +Create the environment dictionary. +[clinic start generated code]*/ + +static PyObject * +os__create_environ_impl(PyObject *module) +/*[clinic end generated code: output=19d9039ab14f8ad4 input=a4c05686b34635e8]*/ +{ + return convertenviron(); +} + + static PyMethodDef posix_methods[] = { OS_STAT_METHODDEF @@ -17023,6 +17037,7 @@ static PyMethodDef posix_methods[] = { OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF OS__INPUTHOOK_METHODDEF OS__IS_INPUTHOOK_INSTALLED_METHODDEF + OS__CREATE_ENVIRON_METHODDEF {NULL, NULL} /* Sentinel */ }; From 6efe3460693c4f39de198a64cebeeee8b1d4e8b6 Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 11 Jun 2024 00:45:16 +0800 Subject: [PATCH 434/903] Fix the CODEOWNERS for _interpretersmodule.c (gh-120288) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 811b8cfdab17dc..8bc40fcb9e8999 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -245,7 +245,7 @@ Doc/howto/clinic.rst @erlend-aasland **/*interpreteridobject.* @ericsnowcurrently **/*crossinterp* @ericsnowcurrently Lib/test/support/interpreters/ @ericsnowcurrently -Modules/_xx*interp*module.c @ericsnowcurrently +Modules/_interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently # Android From 422c4fc855afd18bcc6415902ea1d85a50cb7ce1 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 11 Jun 2024 07:41:12 +0200 Subject: [PATCH 435/903] gh-119600: mock: do not access attributes of original when new_callable is set (#119601) In order to patch flask.g e.g. as in #84982, that proxies getattr must not be invoked. For that, mock must not try to read from the original object. In some cases that is unavoidable, e.g. when doing autospec. However, patch("flask.g", new_callable=MagicMock) should be entirely safe. --- Lib/test/test_unittest/testmock/support.py | 11 +++++++++++ Lib/test/test_unittest/testmock/testpatch.py | 7 +++++++ Lib/unittest/mock.py | 14 +++++++++----- .../2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst | 2 ++ 4 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst diff --git a/Lib/test/test_unittest/testmock/support.py b/Lib/test/test_unittest/testmock/support.py index 49986d65dc47af..6c535b7944f261 100644 --- a/Lib/test/test_unittest/testmock/support.py +++ b/Lib/test/test_unittest/testmock/support.py @@ -14,3 +14,14 @@ def wibble(self): pass class X(object): pass + +# A standin for weurkzeug.local.LocalProxy - issue 119600 +def _inaccessible(*args, **kwargs): + raise AttributeError + + +class OpaqueProxy: + __getattribute__ = _inaccessible + + +g = OpaqueProxy() diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index be75fda7826af1..f26e74ce0bc1ba 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -2045,6 +2045,13 @@ def test(): pass with self.assertRaises(TypeError): test() + def test_patch_proxy_object(self): + @patch("test.test_unittest.testmock.support.g", new_callable=MagicMock()) + def test(_): + pass + + test() + if __name__ == '__main__': unittest.main() diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 3ef83e263f53b7..edabb4520c13cd 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1508,13 +1508,12 @@ def __enter__(self): if isinstance(original, type): # If we're patching out a class and there is a spec inherit = True - if spec is None and _is_async_obj(original): - Klass = AsyncMock - else: - Klass = MagicMock - _kwargs = {} + + # Determine the Klass to use if new_callable is not None: Klass = new_callable + elif spec is None and _is_async_obj(original): + Klass = AsyncMock elif spec is not None or spec_set is not None: this_spec = spec if spec_set is not None: @@ -1527,7 +1526,12 @@ def __enter__(self): Klass = AsyncMock elif not_callable: Klass = NonCallableMagicMock + else: + Klass = MagicMock + else: + Klass = MagicMock + _kwargs = {} if spec is not None: _kwargs['spec'] = spec if spec_set is not None: diff --git a/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst b/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst new file mode 100644 index 00000000000000..04c9ca9c3fd737 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst @@ -0,0 +1,2 @@ +Fix :func:`unittest.mock.patch` to not read attributes of the target when +``new_callable`` is set. Patch by Robert Collins. From 9e9ee50421c857b443e2060274f17fb884d54473 Mon Sep 17 00:00:00 2001 From: blhsing Date: Tue, 11 Jun 2024 13:42:49 +0800 Subject: [PATCH 436/903] gh-65454: avoid triggering call to a PropertyMock in NonCallableMock.__setattr__ (#120019) --- Lib/test/test_unittest/testmock/testhelpers.py | 8 ++++++++ Lib/unittest/mock.py | 3 +++ .../Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst | 1 + 3 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index 74785a83757a92..c9c20f008ca5a2 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -1127,6 +1127,14 @@ def test_propertymock_side_effect(self): p.assert_called_once_with() + def test_propertymock_attach(self): + m = Mock() + p = PropertyMock() + type(m).foo = p + m.attach_mock(p, 'foo') + self.assertEqual(m.mock_calls, []) + + class TestCallablePredicate(unittest.TestCase): def test_type(self): diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index edabb4520c13cd..08975e0e1bd132 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -830,6 +830,9 @@ def __setattr__(self, name, value): mock_name = f'{self._extract_mock_name()}.{name}' raise AttributeError(f'Cannot set {mock_name}') + if isinstance(value, PropertyMock): + self.__dict__[name] = value + return return object.__setattr__(self, name, value) diff --git a/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst b/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst new file mode 100644 index 00000000000000..0b232cf8ca1baf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst @@ -0,0 +1 @@ +:func:`unittest.mock.Mock.attach_mock` no longer triggers a call to a ``PropertyMock`` being attached. From 141babad9b4eceb83371bf19ba3a36b50dd05250 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 11 Jun 2024 10:04:27 +0300 Subject: [PATCH 437/903] gh-120298: Fix use-after-free in `list_richcompare_impl` (#120303) Co-authored-by: Serhiy Storchaka --- Lib/test/test_list.py | 11 +++++++++++ .../2024-06-10-10-42-48.gh-issue-120298.napREA.rst | 2 ++ Objects/listobject.c | 9 ++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 0601b33e79ebb6..d21429fae09b37 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -234,6 +234,17 @@ def __eq__(self, other): list4 = [1] self.assertFalse(list3 == list4) + def test_lt_operator_modifying_operand(self): + # See gh-120298 + class evil: + def __lt__(self, other): + other.clear() + return NotImplemented + + a = [[evil()]] + with self.assertRaises(TypeError): + a[0] < a + @cpython_only def test_preallocation(self): iterable = [0] * 10 diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst new file mode 100644 index 00000000000000..531d39517ac423 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst @@ -0,0 +1,2 @@ +Fix use-after free in ``list_richcompare_impl`` which can be invoked via +some specificly tailored evil input. diff --git a/Objects/listobject.c b/Objects/listobject.c index d09bb6391034d1..6829d5d28656cf 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3382,7 +3382,14 @@ list_richcompare_impl(PyObject *v, PyObject *w, int op) } /* Compare the final item again using the proper operator */ - return PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); + PyObject *vitem = vl->ob_item[i]; + PyObject *witem = wl->ob_item[i]; + Py_INCREF(vitem); + Py_INCREF(witem); + PyObject *result = PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); + Py_DECREF(vitem); + Py_DECREF(witem); + return result; } static PyObject * From 7d2447137e117ea9a6ee1493bce0b071c76b1bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 11 Jun 2024 09:11:13 +0200 Subject: [PATCH 438/903] gh-120291: Fix a bashism in python-config.sh.in (#120292) gh-120291: Fix bashisms in python-config.sh.in Replace the use of bash-specific `[[ ... ]]` with POSIX-compliant `[ ... ]` to make the `python-config` shell script work with non-bash shells again. While at it, use `local` in a safer way, since it is not in POSIX either (though universally supported). Fixes #120291 --- .../Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst | 1 + Misc/python-config.sh.in | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst diff --git a/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst b/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst new file mode 100644 index 00000000000000..d0bb297b51dc6e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst @@ -0,0 +1 @@ +Make the ``python-config`` shell script compatible with non-bash shells. diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index c3c0b34fc1451d..9929f5b2653dca 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -4,11 +4,12 @@ exit_with_usage () { - local USAGE="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" - if [[ "$1" -eq 0 ]]; then - echo "$USAGE" + local usage + usage="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + if [ "$1" -eq 0 ]; then + echo "$usage" else - echo "$USAGE" >&2 + echo "$usage" >&2 fi exit $1 } From 02c1dfff073a3dd6ce34a11b038defde291c2203 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Tue, 11 Jun 2024 10:56:38 +0300 Subject: [PATCH 439/903] gh-120080: Mark test_round_with_none_arg_direct_call as cpython_only (#120328) --- Lib/test/test_float.py | 1 + Lib/test/test_int.py | 1 + 2 files changed, 2 insertions(+) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 53695cefb8fded..756cf9bd7719c0 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -949,6 +949,7 @@ def test_None_ndigits(self): self.assertEqual(x, 2) self.assertIsInstance(x, int) + @support.cpython_only def test_round_with_none_arg_direct_call(self): for val in [(1.0).__round__(None), round(1.0), diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 77221dfb6d5aa2..2747d9219255ac 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -517,6 +517,7 @@ def test_issue31619(self): self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + @support.cpython_only def test_round_with_none_arg_direct_call(self): for val in [(1).__round__(None), round(1), From 9b8611eeea172cd4aa626ccd1ca333dc4093cd8c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 11 Jun 2024 07:06:49 -0600 Subject: [PATCH 440/903] gh-119180: PEP 649 compiler changes (#119361) --- .../pycore_global_objects_fini_generated.h | 2 +- Include/internal/pycore_global_strings.h | 2 +- Include/internal/pycore_opcode_utils.h | 1 + .../internal/pycore_runtime_init_generated.h | 2 +- Include/internal/pycore_symtable.h | 8 +- .../internal/pycore_unicodeobject_generated.h | 3 - Lib/inspect.py | 8 +- Lib/symtable.py | 2 + Lib/test/test_dis.py | 29 +- Lib/test/test_grammar.py | 69 +--- Lib/test/test_module/__init__.py | 2 + Lib/test/test_opcodes.py | 13 +- Lib/test/test_positional_only_arg.py | 5 +- Lib/test/test_pyclbr.py | 2 + Lib/test/test_pydoc/test_pydoc.py | 11 +- Lib/test/test_pyrepl/test_interact.py | 2 +- Lib/test/test_symtable.py | 6 +- Lib/test/test_traceback.py | 5 +- Lib/test/test_type_annotations.py | 163 +++++++- Lib/test/test_typing.py | 4 +- Lib/test/typinganndata/ann_module.py | 4 - Lib/typing.py | 23 +- ...-05-22-06-22-47.gh-issue-119180.vZMiXm.rst | 1 + Python/bytecodes.c | 5 + Python/compile.c | 357 ++++++++++-------- Python/executor_cases.c.h | 5 + Python/generated_cases.c.h | 5 + Python/symtable.c | 198 +++++++--- 28 files changed, 609 insertions(+), 328 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index b186408931c92e..30851dc2dbec44 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -559,6 +559,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); @@ -745,7 +746,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index e1808c85acfb2d..009802c441685c 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -45,6 +45,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(dot, ".") STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") + STRUCT_FOR_STR(format, ".format") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") @@ -234,7 +235,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_abstract_) STRUCT_FOR_ID(_active) STRUCT_FOR_ID(_align_) - STRUCT_FOR_ID(_annotation) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index b06e469dd5bd91..e76f4840a66891 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -57,6 +57,7 @@ extern "C" { #define MAKE_FUNCTION_KWDEFAULTS 0x02 #define MAKE_FUNCTION_ANNOTATIONS 0x04 #define MAKE_FUNCTION_CLOSURE 0x08 +#define MAKE_FUNCTION_ANNOTATE 0x10 /* Values used as the oparg for LOAD_COMMON_CONSTANT */ #define CONSTANT_ASSERTIONERROR 0 diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 2dde6febc2cae4..ff5b6ee8e0f006 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -554,6 +554,7 @@ extern "C" { INIT_STR(dot, "."), \ INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ + INIT_STR(format, ".format"), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ @@ -743,7 +744,6 @@ extern "C" { INIT_ID(_abstract_), \ INIT_ID(_active), \ INIT_ID(_align_), \ - INIT_ID(_annotation), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index ac6c499c08264e..5d544765237df5 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -12,8 +12,9 @@ struct _mod; // Type defined in pycore_ast.h typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, - // Used for annotations if 'from __future__ import annotations' is active. - // Annotation blocks cannot bind names and are not evaluated. + // Used for annotations. If 'from __future__ import annotations' is active, + // annotation blocks cannot bind names and are not evaluated. Otherwise, they + // are lazily evaluated (see PEP 649). AnnotationBlock, // Used for generics and type aliases. These work mostly like functions // (see PEP 695 for details). The three different blocks function identically; @@ -89,6 +90,7 @@ typedef struct _symtable_entry { including free refs to globals */ unsigned ste_generator : 1; /* true if namespace is a generator */ unsigned ste_coroutine : 1; /* true if namespace is a coroutine */ + unsigned ste_annotations_used : 1; /* true if there are any annotations in this scope */ _Py_comprehension_ty ste_comprehension; /* Kind of comprehension (if any) */ unsigned ste_varargs : 1; /* true if block has varargs */ unsigned ste_varkeywords : 1; /* true if block has varkeywords */ @@ -110,6 +112,7 @@ typedef struct _symtable_entry { int ste_end_col_offset; /* end offset of first line of block */ int ste_opt_lineno; /* lineno of last exec or import * */ int ste_opt_col_offset; /* offset of last exec or import * */ + struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ struct symtable *ste_table; } PySTEntryObject; @@ -126,6 +129,7 @@ extern struct symtable* _PySymtable_Build( PyObject *filename, _PyFutureFeatures *future); extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); +extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **); extern void _PySymtable_Free(struct symtable *); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b00119a1bad7ff..69d93a9610a2e5 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -543,9 +543,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_align_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_annotation); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_anonymous_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/inspect.py b/Lib/inspect.py index 2b7f8bec482f8e..5570a43ebfea19 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -220,13 +220,7 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False): """ if isinstance(obj, type): # class - obj_dict = getattr(obj, '__dict__', None) - if obj_dict and hasattr(obj_dict, 'get'): - ann = obj_dict.get('__annotations__', None) - if isinstance(ann, types.GetSetDescriptorType): - ann = None - else: - ann = None + ann = obj.__annotations__ obj_globals = None module_name = getattr(obj, '__module__', None) diff --git a/Lib/symtable.py b/Lib/symtable.py index ba2f0dafcd0063..af65e93e68eda4 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -222,6 +222,8 @@ def get_methods(self): if self.__methods is None: d = {} for st in self._table.children: + if st.type == _symtable.TYPE_ANNOTATION: + continue d[st.name] = 1 self.__methods = tuple(d) return self.__methods diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index b1a1b77c53e8cb..b0ae1289224070 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -352,32 +352,21 @@ def wrap_func_w_kwargs(): dis_annot_stmt_str = """\ 0 RESUME 0 - 2 SETUP_ANNOTATIONS - LOAD_CONST 0 (1) + 2 LOAD_CONST 0 (1) STORE_NAME 0 (x) - LOAD_NAME 1 (int) - LOAD_NAME 2 (__annotations__) - LOAD_CONST 1 ('x') - STORE_SUBSCR - - 3 LOAD_NAME 3 (fun) - PUSH_NULL - LOAD_CONST 0 (1) - CALL 1 - LOAD_NAME 2 (__annotations__) - LOAD_CONST 2 ('y') - STORE_SUBSCR 4 LOAD_CONST 0 (1) - LOAD_NAME 4 (lst) - LOAD_NAME 3 (fun) + LOAD_NAME 1 (lst) + LOAD_NAME 2 (fun) PUSH_NULL - LOAD_CONST 3 (0) + LOAD_CONST 1 (0) CALL 1 STORE_SUBSCR - LOAD_NAME 1 (int) - POP_TOP - RETURN_CONST 4 (None) + + 2 LOAD_CONST 2 (", line 2>) + MAKE_FUNCTION + STORE_NAME 3 (__annotate__) + RETURN_CONST 3 (None) """ compound_stmt_str = """\ diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index c72f4387108ca8..5b7a639c025a0f 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -306,16 +306,6 @@ def test_eof_error(self): var_annot_global: int # a global annotated is necessary for test_var_annot -# custom namespace for testing __annotations__ - -class CNS: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - self._dct[item.lower()] = value - def __getitem__(self, item): - return self._dct[item] - class GrammarTests(unittest.TestCase): @@ -446,22 +436,12 @@ class F(C, A): self.assertEqual(E.__annotations__, {}) self.assertEqual(F.__annotations__, {}) - - def test_var_annot_metaclass_semantics(self): - class CMeta(type): - @classmethod - def __prepare__(metacls, name, bases, **kwds): - return {'__annotations__': CNS()} - class CC(metaclass=CMeta): - XX: 'ANNOT' - self.assertEqual(CC.__annotations__['xx'], 'ANNOT') - def test_var_annot_module_semantics(self): self.assertEqual(test.__annotations__, {}) self.assertEqual(ann_module.__annotations__, - {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) + {'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) self.assertEqual(ann_module.M.__annotations__, - {'123': 123, 'o': type}) + {'o': type}) self.assertEqual(ann_module2.__annotations__, {}) def test_var_annot_in_module(self): @@ -476,51 +456,12 @@ def test_var_annot_in_module(self): ann_module3.D_bad_ann(5) def test_var_annot_simple_exec(self): - gns = {}; lns= {} + gns = {}; lns = {} exec("'docstring'\n" - "__annotations__[1] = 2\n" "x: int = 5\n", gns, lns) - self.assertEqual(lns["__annotations__"], {1: 2, 'x': int}) - with self.assertRaises(KeyError): - gns['__annotations__'] - - def test_var_annot_custom_maps(self): - # tests with custom locals() and __annotations__ - ns = {'__annotations__': CNS()} - exec('X: int; Z: str = "Z"; (w): complex = 1j', ns) - self.assertEqual(ns['__annotations__']['x'], int) - self.assertEqual(ns['__annotations__']['z'], str) + self.assertEqual(lns["__annotate__"](1), {'x': int}) with self.assertRaises(KeyError): - ns['__annotations__']['w'] - nonloc_ns = {} - class CNS2: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('x: int = 1', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], int) - - def test_var_annot_refleak(self): - # complex case: custom locals plus custom __annotations__ - # this was causing refleak - cns = CNS() - nonloc_ns = {'__annotations__': cns} - class CNS2: - def __init__(self): - self._dct = {'__annotations__': cns} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('X: str', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], str) + gns['__annotate__'] def test_var_annot_rhs(self): ns = {} diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index 952ba43f72504d..56edd0c637f376 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -360,6 +360,8 @@ def test_annotations_are_created_correctly(self): ann_module4 = import_helper.import_fresh_module( 'test.typinganndata.ann_module4', ) + self.assertFalse("__annotations__" in ann_module4.__dict__) + self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 72488b2bb6b4ff..f7cc8331b8d844 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -39,16 +39,19 @@ class C: pass def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) - self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + self.assertEqual(ns['__annotations__'], {1: 2}) def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): - del globals()['__annotations__'] + globals().pop('__annotations__', None) class C: - del __annotations__ - with self.assertRaises(NameError): - x: int + try: + del __annotations__ + except NameError: + pass + x: int + self.assertEqual(C.__annotations__, {"x": int}) def test_raise_class_exceptions(self): diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index 1a193814d7535d..eea0625012da6d 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -2,6 +2,7 @@ import dis import pickle +import types import unittest from test.support import check_syntax_error @@ -440,7 +441,9 @@ def f(x: not (int is int), /): ... # without constant folding we end up with # COMPARE_OP(is), IS_OP (0) # with constant folding we should expect a IS_OP (1) - codes = [(i.opname, i.argval) for i in dis.get_instructions(g)] + code_obj = next(const for const in g.__code__.co_consts + if isinstance(const, types.CodeType) and const.co_name == "__annotate__") + codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)] self.assertNotIn(('UNARY_NOT', None), codes) self.assertIn(('IS_OP', 1), codes) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 46206accbafc36..0c12a3085b12af 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -109,6 +109,8 @@ def ismethod(oclass, obj, name): actualMethods = [] for m in py_item.__dict__.keys(): + if m == "__annotate__": + continue if ismethod(py_item, getattr(py_item, m), m): actualMethods.append(m) foundMethods = [] diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 57e5b8e8abddfa..a17c16cc73cf0e 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -77,6 +77,11 @@ class A(builtins.object) | __weakref__%s class B(builtins.object) + | Methods defined here: + | + | __annotate__(...) + | + | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__%s @@ -87,8 +92,6 @@ class B(builtins.object) | Data and other attributes defined here: | | NO_MEANING = 'eggs' - | - | __annotations__ = {'NO_MEANING': } class C(builtins.object) | Methods defined here: @@ -176,6 +179,9 @@ class A(builtins.object) list of weak references to the object class B(builtins.object) + Methods defined here: + __annotate__(...) + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ dictionary for instance variables @@ -184,7 +190,6 @@ class B(builtins.object) ---------------------------------------------------------------------- Data and other attributes defined here: NO_MEANING = 'eggs' - __annotations__ = {'NO_MEANING': } class C(builtins.object) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index df97b1354a168e..31f08cdb25e078 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -105,7 +105,7 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self): def test_no_active_future(self): console = InteractiveColoredConsole() - source = "x: int = 1; print(__annotations__)" + source = "x: int = 1; print(__annotate__(1))" f = io.StringIO() with contextlib.redirect_stdout(f): result = console.runsource(source) diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ef2a228b15ed4e..a4b111e865c86e 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -205,12 +205,14 @@ def test_assigned(self): def test_annotated(self): st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') - st2 = st1.get_children()[0] + st2 = st1.get_children()[1] + self.assertEqual(st2.get_type(), "function") self.assertTrue(st2.lookup('x').is_local()) self.assertTrue(st2.lookup('x').is_annotated()) self.assertFalse(st2.lookup('x').is_global()) st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') - st4 = st3.get_children()[0] + st4 = st3.get_children()[1] + self.assertEqual(st4.get_type(), "function") self.assertTrue(st4.lookup('x').is_local()) self.assertFalse(st4.lookup('x').is_annotated()) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 5035de114b5e9d..1895c88d23b70d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -622,6 +622,7 @@ def test_caret_in_type_annotation(self): def f_with_type(): def foo(a: THIS_DOES_NOT_EXIST ) -> int: return 0 + foo.__annotations__ lineno_f = f_with_type.__code__.co_firstlineno expected_f = ( @@ -629,7 +630,9 @@ def foo(a: THIS_DOES_NOT_EXIST ) -> int: f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' ' ~~~~~~~~^^\n' - f' File "{__file__}", line {lineno_f+1}, in f_with_type\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_type\n' + ' foo.__annotations__\n' + f' File "{__file__}", line {lineno_f+1}, in __annotate__\n' ' def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n' ' ^^^^^^^^^^^^^^^^^^^\n' ) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 5e3c3347a41571..a9be1f5aa84681 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,7 +1,12 @@ import textwrap import types import unittest -from test.support import run_code +from test.support import run_code, check_syntax_error + +VALUE = 1 +FORWARDREF = 2 +SOURCE = 3 + class TypeAnnotationTests(unittest.TestCase): @@ -49,6 +54,7 @@ def test_annotations_are_created_correctly(self): class C: a:int=3 b:str=4 + self.assertEqual(C.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in C.__dict__) del C.__annotations__ self.assertFalse("__annotations__" in C.__dict__) @@ -106,6 +112,13 @@ class D(metaclass=C): self.assertEqual(D.__annotations__, {}) +def build_module(code: str, name: str = "top") -> types.ModuleType: + ns = run_code(code) + mod = types.ModuleType(name) + mod.__dict__.update(ns) + return mod + + class TestSetupAnnotations(unittest.TestCase): def check(self, code: str): code = textwrap.dedent(code) @@ -113,11 +126,10 @@ def check(self, code: str): with self.subTest(scope=scope): if scope == "class": code = f"class C:\n{textwrap.indent(code, ' ')}" - ns = run_code(code) - if scope == "class": + ns = run_code(code) annotations = ns["C"].__annotations__ else: - annotations = ns["__annotations__"] + annotations = build_module(code).__annotations__ self.assertEqual(annotations, {"x": int}) def test_top_level(self): @@ -256,3 +268,146 @@ def check_annotations(self, f): # Setting f.__annotations__ also clears __annotate__ f.__annotations__ = {"z": 43} self.assertIs(f.__annotate__, None) + + +class DeferredEvaluationTests(unittest.TestCase): + def test_function(self): + def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_async_function(self): + async def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_class(self): + class X: + a: undefined + + with self.assertRaises(NameError): + X.__annotations__ + + undefined = 1 + self.assertEqual(X.__annotations__, {"a": 1}) + + def test_module(self): + ns = run_code("x: undefined = 1") + anno = ns["__annotate__"] + with self.assertRaises(NotImplementedError): + anno(2) + + with self.assertRaises(NameError): + anno(1) + + ns["undefined"] = 1 + self.assertEqual(anno(1), {"x": 1}) + + def test_class_scoping(self): + class Outer: + def meth(self, x: Nested): ... + x: Nested + class Nested: ... + + self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested}) + self.assertEqual(Outer.__annotations__, {"x": Outer.Nested}) + + def test_no_exotic_expressions(self): + check_syntax_error(self, "def func(x: (yield)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (yield from x)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (y := 3)): ...", "named expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (await 42)): ...", "await expression cannot be used within an annotation") + + def test_no_exotic_expressions_in_unevaluated_annotations(self): + preludes = [ + "", + "class X: ", + "def f(): ", + "async def f(): ", + ] + for prelude in preludes: + with self.subTest(prelude=prelude): + check_syntax_error(self, prelude + "(x): (yield)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (yield from x)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (y := 3)", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (await 42)", "await expression cannot be used within an annotation") + + def test_ignore_non_simple_annotations(self): + ns = run_code("class X: (y): int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int.b: int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int[str]: int") + self.assertEqual(ns["X"].__annotations__, {}) + + def test_generated_annotate(self): + def func(x: int): + pass + class X: + x: int + mod = build_module("x: int") + for obj in (func, X, mod): + with self.subTest(obj=obj): + annotate = obj.__annotate__ + self.assertIsInstance(annotate, types.FunctionType) + self.assertEqual(annotate.__name__, "__annotate__") + with self.assertRaises(NotImplementedError): + annotate(FORWARDREF) + with self.assertRaises(NotImplementedError): + annotate(SOURCE) + with self.assertRaises(NotImplementedError): + annotate(None) + self.assertEqual(annotate(VALUE), {"x": int}) + + def test_comprehension_in_annotation(self): + # This crashed in an earlier version of the code + ns = run_code("x: [y for y in range(10)]") + self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + + def f(x: int) -> int: pass + """ + ns = run_code(code) + f = ns["f"] + self.assertIsInstance(f.__annotate__, types.FunctionType) + annos = {"x": "int", "return": "int"} + self.assertEqual(f.__annotate__(VALUE), annos) + self.assertEqual(f.__annotations__, annos) + + def test_name_clash_with_format(self): + # this test would fail if __annotate__'s parameter was called "format" + code = """ + class format: pass + + def f(x: format): pass + """ + ns = run_code(code) + f = ns["f"] + self.assertEqual(f.__annotations__, {"x": ns["format"]}) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index dac55ceb9e99e0..9800b3b6a7da29 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6634,7 +6634,7 @@ def test_get_type_hints_from_various_objects(self): gth(None) def test_get_type_hints_modules(self): - ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} + ann_module_type_hints = {'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} self.assertEqual(gth(ann_module), ann_module_type_hints) self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) @@ -6652,7 +6652,7 @@ def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) self.assertIsInstance(gth(ann_module.j_class), dict) - self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type}) + self.assertEqual(gth(ann_module.M), {'o': type}) self.assertEqual(gth(ann_module.D), {'j': str, 'k': str, 'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.Y), {'z': int}) diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py index 5081e6b58345a9..e1a1792cb4a867 100644 --- a/Lib/test/typinganndata/ann_module.py +++ b/Lib/test/typinganndata/ann_module.py @@ -8,8 +8,6 @@ from typing import Optional from functools import wraps -__annotations__[1] = 2 - class C: x = 5; y: Optional['C'] = None @@ -18,8 +16,6 @@ class C: x: int = 5; y: str = x; f: Tuple[int, int] class M(type): - - __annotations__['123'] = 123 o: type = object (pars): bool = True diff --git a/Lib/typing.py b/Lib/typing.py index be49aa63464f05..7a9149d3f3c2c1 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2412,7 +2412,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) + ann = getattr(base, '__annotations__', {}) if isinstance(ann, types.GetSetDescriptorType): ann = {} base_locals = dict(vars(base)) if localns is None else localns @@ -2970,7 +2970,12 @@ def __new__(cls, typename, bases, ns): raise TypeError( 'can only inherit from a NamedTuple type and Generic') bases = tuple(tuple if base is _NamedTuple else base for base in bases) - types = ns.get('__annotations__', {}) + if "__annotations__" in ns: + types = ns["__annotations__"] + elif "__annotate__" in ns: + types = ns["__annotate__"](1) # VALUE + else: + types = {} default_names = [] for field_name in types: if field_name in ns: @@ -3131,7 +3136,12 @@ def __new__(cls, name, bases, ns, total=True): tp_dict.__orig_bases__ = bases annotations = {} - own_annotations = ns.get('__annotations__', {}) + if "__annotations__" in ns: + own_annotations = ns["__annotations__"] + elif "__annotate__" in ns: + own_annotations = ns["__annotate__"](1) # VALUE + else: + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" own_annotations = { n: _type_check(tp, msg, module=tp_dict.__module__) @@ -3143,7 +3153,12 @@ def __new__(cls, name, bases, ns, total=True): mutable_keys = set() for base in bases: - annotations.update(base.__dict__.get('__annotations__', {})) + # TODO: Avoid eagerly evaluating annotations in VALUE format. + # Instead, evaluate in FORWARDREF format to figure out which + # keys have Required/NotRequired/ReadOnly qualifiers, and create + # a new __annotate__ function for the resulting TypedDict that + # combines the annotations from this class and its parents. + annotations.update(base.__annotations__) base_required = base.__dict__.get('__required_keys__', set()) required_keys |= base_required diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst new file mode 100644 index 00000000000000..265ffb32e6a1f9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst @@ -0,0 +1 @@ +Evaluation of annotations is now deferred. See :pep:`649` for details. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 413ad1105f9428..05c17ac334b69f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3975,6 +3975,11 @@ dummy_func( assert(func_obj->func_defaults == NULL); func_obj->func_defaults = attr; break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: Py_UNREACHABLE(); } diff --git a/Python/compile.c b/Python/compile.c index cb724154206b7e..c3372766d0bd50 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -132,7 +132,7 @@ enum { COMPILER_SCOPE_ASYNC_FUNCTION, COMPILER_SCOPE_LAMBDA, COMPILER_SCOPE_COMPREHENSION, - COMPILER_SCOPE_TYPEPARAMS, + COMPILER_SCOPE_ANNOTATIONS, }; @@ -142,6 +142,15 @@ typedef _PyInstructionSequence instr_sequence; #define INITIAL_INSTR_SEQUENCE_SIZE 100 #define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 +static const int compare_masks[] = { + [Py_LT] = COMPARISON_LESS_THAN, + [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, + [Py_EQ] = COMPARISON_EQUALS, + [Py_NE] = COMPARISON_NOT_EQUALS, + [Py_GT] = COMPARISON_GREATER_THAN, + [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS, +}; + /* * Resize the array if index is out of range. * @@ -208,6 +217,7 @@ struct compiler_unit { PyObject *u_private; /* for private name mangling */ PyObject *u_static_attributes; /* for class: attributes accessed via self.X */ + PyObject *u_deferred_annotations; /* AnnAssign nodes deferred to the end of compilation */ instr_sequence *u_instr_sequence; /* codegen output */ @@ -330,6 +340,8 @@ static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *); static int compiler_match(struct compiler *, stmt_ty); static int compiler_pattern_subpattern(struct compiler *, pattern_ty, pattern_context *); +static int compiler_make_closure(struct compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags); static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone); @@ -545,6 +557,7 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); Py_CLEAR(u->u_static_attributes); + Py_CLEAR(u->u_deferred_annotations); PyMem_Free(u); } @@ -582,8 +595,8 @@ compiler_set_qualname(struct compiler *c) capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1); parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME); assert(parent); - if (parent->u_scope_type == COMPILER_SCOPE_TYPEPARAMS) { - /* The parent is a type parameter scope, so we need to + if (parent->u_scope_type == COMPILER_SCOPE_ANNOTATIONS) { + /* The parent is an annotation scope, so we need to look at the grandparent. */ if (stack_size == 2) { // If we're immediately within the module, we can skip @@ -1128,6 +1141,7 @@ compiler_enter_scope(struct compiler *c, identifier name, } u->u_private = NULL; + u->u_deferred_annotations = NULL; if (scope_type == COMPILER_SCOPE_CLASS) { u->u_static_attributes = PySet_New(0); if (!u->u_static_attributes) { @@ -1209,85 +1223,6 @@ compiler_exit_scope(struct compiler *c) PyErr_SetRaisedException(exc); } -/* Search if variable annotations are present statically in a block. */ - -static bool -find_ann(asdl_stmt_seq *stmts) -{ - int i, j, res = 0; - stmt_ty st; - - for (i = 0; i < asdl_seq_LEN(stmts); i++) { - st = (stmt_ty)asdl_seq_GET(stmts, i); - switch (st->kind) { - case AnnAssign_kind: - return true; - case For_kind: - res = find_ann(st->v.For.body) || - find_ann(st->v.For.orelse); - break; - case AsyncFor_kind: - res = find_ann(st->v.AsyncFor.body) || - find_ann(st->v.AsyncFor.orelse); - break; - case While_kind: - res = find_ann(st->v.While.body) || - find_ann(st->v.While.orelse); - break; - case If_kind: - res = find_ann(st->v.If.body) || - find_ann(st->v.If.orelse); - break; - case With_kind: - res = find_ann(st->v.With.body); - break; - case AsyncWith_kind: - res = find_ann(st->v.AsyncWith.body); - break; - case Try_kind: - for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - st->v.Try.handlers, j); - if (find_ann(handler->v.ExceptHandler.body)) { - return true; - } - } - res = find_ann(st->v.Try.body) || - find_ann(st->v.Try.finalbody) || - find_ann(st->v.Try.orelse); - break; - case TryStar_kind: - for (j = 0; j < asdl_seq_LEN(st->v.TryStar.handlers); j++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - st->v.TryStar.handlers, j); - if (find_ann(handler->v.ExceptHandler.body)) { - return true; - } - } - res = find_ann(st->v.TryStar.body) || - find_ann(st->v.TryStar.finalbody) || - find_ann(st->v.TryStar.orelse); - break; - case Match_kind: - for (j = 0; j < asdl_seq_LEN(st->v.Match.cases); j++) { - match_case_ty match_case = (match_case_ty)asdl_seq_GET( - st->v.Match.cases, j); - if (find_ann(match_case->body)) { - return true; - } - } - break; - default: - res = false; - break; - } - if (res) { - break; - } - } - return res; -} - /* * Frame block handling functions */ @@ -1502,6 +1437,47 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, return SUCCESS; } +static int +compiler_setup_annotations_scope(struct compiler *c, location loc, + void *key, PyObject *name) +{ + if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, + key, loc.lineno) == -1) { + return ERROR; + } + c->u->u_metadata.u_posonlyargcount = 1; + // if .format != 1: raise NotImplementedError + _Py_DECLARE_STR(format, ".format"); + ADDOP_I(c, loc, LOAD_FAST, 0); + ADDOP_LOAD_CONST(c, loc, _PyLong_GetOne()); + ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]); + NEW_JUMP_TARGET_LABEL(c, body); + ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); + ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); + ADDOP_I(c, loc, RAISE_VARARGS, 1); + USE_LABEL(c, body); + return 0; +} + +static int +compiler_leave_annotations_scope(struct compiler *c, location loc, + Py_ssize_t annotations_len) +{ + ADDOP_I(c, loc, BUILD_MAP, annotations_len); + ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); + PyCodeObject *co = optimize_and_assemble(c, 1); + compiler_exit_scope(c); + if (co == NULL) { + return ERROR; + } + if (compiler_make_closure(c, loc, co, 0) < 0) { + Py_DECREF(co); + return ERROR; + } + Py_DECREF(co); + return 0; +} + /* Compile a sequence of statements, checking for a docstring and for annotations. */ @@ -1517,34 +1493,79 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); loc = LOC(st); } - /* Every annotated class and module should have __annotations__. */ - if (find_ann(stmts)) { + /* If from __future__ import annotations is active, + * every annotated class and module should have __annotations__. + * Else __annotate__ is created when necessary. */ + if ((c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && c->u->u_ste->ste_annotations_used) { ADDOP(c, loc, SETUP_ANNOTATIONS); } if (!asdl_seq_LEN(stmts)) { return SUCCESS; } Py_ssize_t first_instr = 0; - PyObject *docstring = _PyAST_GetDocString(stmts); - if (docstring) { - first_instr = 1; - /* if not -OO mode, set docstring */ - if (c->c_optimize < 2) { - PyObject *cleandoc = _PyCompile_CleanDoc(docstring); - if (cleandoc == NULL) { - return ERROR; + if (!c->c_interactive) { + PyObject *docstring = _PyAST_GetDocString(stmts); + if (docstring) { + first_instr = 1; + /* if not -OO mode, set docstring */ + if (c->c_optimize < 2) { + PyObject *cleandoc = _PyCompile_CleanDoc(docstring); + if (cleandoc == NULL) { + return ERROR; + } + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); + assert(st->kind == Expr_kind); + location loc = LOC(st->v.Expr.value); + ADDOP_LOAD_CONST(c, loc, cleandoc); + Py_DECREF(cleandoc); + RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } - stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); - assert(st->kind == Expr_kind); - location loc = LOC(st->v.Expr.value); - ADDOP_LOAD_CONST(c, loc, cleandoc); - Py_DECREF(cleandoc); - RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) { VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i)); } + // If there are annotations and the future import is not on, we + // collect the annotations in a separate pass and generate an + // __annotate__ function. See PEP 649. + if (!(c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && + c->u->u_deferred_annotations != NULL) { + + // It's possible that ste_annotations_block is set but + // u_deferred_annotations is not, because the former is still + // set if there are only non-simple annotations (i.e., annotations + // for attributes, subscripts, or parenthesized names). However, the + // reverse should not be possible. + assert(c->u->u_ste->ste_annotation_block != NULL); + PyObject *deferred_anno = Py_NewRef(c->u->u_deferred_annotations); + void *key = (void *)((uintptr_t)c->u->u_ste->ste_id + 1); + if (compiler_setup_annotations_scope(c, loc, key, + c->u->u_ste->ste_annotation_block->ste_name) == -1) { + Py_DECREF(deferred_anno); + return ERROR; + } + Py_ssize_t annotations_len = PyList_Size(deferred_anno); + for (Py_ssize_t i = 0; i < annotations_len; i++) { + PyObject *ptr = PyList_GET_ITEM(deferred_anno, i); + stmt_ty st = (stmt_ty)PyLong_AsVoidPtr(ptr); + if (st == NULL) { + compiler_exit_scope(c); + Py_DECREF(deferred_anno); + return ERROR; + } + PyObject *mangled = _Py_Mangle(c->u->u_private, st->v.AnnAssign.target->v.Name.id); + ADDOP_LOAD_CONST_NEW(c, LOC(st), mangled); + VISIT(c, expr, st->v.AnnAssign.annotation); + } + Py_DECREF(deferred_anno); + + RETURN_IF_ERROR( + compiler_leave_annotations_scope(c, loc, annotations_len) + ); + RETURN_IF_ERROR( + compiler_nameop(c, loc, &_Py_ID(__annotate__), Store) + ); + } return SUCCESS; } @@ -1559,11 +1580,10 @@ compiler_codegen(struct compiler *c, mod_ty mod) } break; case Interactive_kind: - if (find_ann(mod->v.Interactive.body)) { - ADDOP(c, loc, SETUP_ANNOTATIONS); - } c->c_interactive = 1; - VISIT_SEQ(c, stmt, mod->v.Interactive.body); + if (compiler_body(c, loc, mod->v.Interactive.body) < 0) { + return ERROR; + } break; case Expression_kind: VISIT(c, expr, mod->v.Expression.body); @@ -1702,6 +1722,9 @@ compiler_make_closure(struct compiler *c, location loc, if (flags & MAKE_FUNCTION_ANNOTATIONS) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS); } + if (flags & MAKE_FUNCTION_ANNOTATE) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE); + } if (flags & MAKE_FUNCTION_KWDEFAULTS) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS); } @@ -1833,7 +1856,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id, VISIT(c, expr, annotation); } } - *annotations_len += 2; + *annotations_len += 1; return SUCCESS; } @@ -1856,43 +1879,76 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args, } static int -compiler_visit_annotations(struct compiler *c, location loc, - arguments_ty args, expr_ty returns) +compiler_visit_annotations_in_scope(struct compiler *c, location loc, + arguments_ty args, expr_ty returns, + Py_ssize_t *annotations_len) { - /* Push arg annotation names and values. - The expressions are evaluated out-of-order wrt the source code. - - Return -1 on error, 0 if no annotations pushed, 1 if a annotations is pushed. - */ - Py_ssize_t annotations_len = 0; - RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->args, &annotations_len, loc)); + compiler_visit_argannotations(c, args->args, annotations_len, loc)); RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc)); + compiler_visit_argannotations(c, args->posonlyargs, annotations_len, loc)); if (args->vararg && args->vararg->annotation) { RETURN_IF_ERROR( compiler_visit_argannotation(c, args->vararg->arg, - args->vararg->annotation, &annotations_len, loc)); + args->vararg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc)); + compiler_visit_argannotations(c, args->kwonlyargs, annotations_len, loc)); if (args->kwarg && args->kwarg->annotation) { RETURN_IF_ERROR( compiler_visit_argannotation(c, args->kwarg->arg, - args->kwarg->annotation, &annotations_len, loc)); + args->kwarg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_visit_argannotation(c, &_Py_ID(return), returns, &annotations_len, loc)); + compiler_visit_argannotation(c, &_Py_ID(return), returns, annotations_len, loc)); - if (annotations_len) { - ADDOP_I(c, loc, BUILD_TUPLE, annotations_len); - return 1; + return 0; +} + +static int +compiler_visit_annotations(struct compiler *c, location loc, + arguments_ty args, expr_ty returns) +{ + /* Push arg annotation names and values. + The expressions are evaluated separately from the rest of the source code. + + Return -1 on error, or a combination of flags to add to the function. + */ + Py_ssize_t annotations_len = 0; + + PySTEntryObject *ste; + if (_PySymtable_LookupOptional(c->c_st, args, &ste) < 0) { + return ERROR; + } + assert(ste != NULL); + bool annotations_used = ste->ste_annotations_used; + + if (annotations_used) { + if (compiler_setup_annotations_scope(c, loc, (void *)args, + ste->ste_name) < 0) { + Py_DECREF(ste); + return ERROR; + } + } + Py_DECREF(ste); + + if (compiler_visit_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) { + if (annotations_used) { + compiler_exit_scope(c); + } + return ERROR; + } + + if (annotations_used) { + RETURN_IF_ERROR( + compiler_leave_annotations_scope(c, loc, annotations_len) + ); + return MAKE_FUNCTION_ANNOTATE; } return 0; @@ -2001,7 +2057,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, identifier name, void *key, bool allow_starred) { - if (compiler_enter_scope(c, name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, key, e->lineno) == -1) { return ERROR; } @@ -2220,7 +2276,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) asdl_expr_seq *decos; asdl_type_param_seq *type_params; Py_ssize_t funcflags; - int annotations; int firstlineno; if (is_async) { @@ -2274,7 +2329,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, (void *)type_params, firstlineno) == -1) { Py_DECREF(type_params_name); return ERROR; @@ -2286,16 +2341,14 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) } } - annotations = compiler_visit_annotations(c, loc, args, returns); - if (annotations < 0) { + int annotations_flag = compiler_visit_annotations(c, loc, args, returns); + if (annotations_flag < 0) { if (is_generic) { compiler_exit_scope(c); } return ERROR; } - if (annotations > 0) { - funcflags |= MAKE_FUNCTION_ANNOTATIONS; - } + funcflags |= annotations_flag; if (compiler_function_body(c, s, is_async, funcflags, firstlineno) < 0) { if (is_generic) { @@ -2510,7 +2563,7 @@ compiler_class(struct compiler *c, stmt_ty s) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, (void *)type_params, firstlineno) == -1) { Py_DECREF(type_params_name); return ERROR; @@ -2630,7 +2683,7 @@ compiler_typealias(struct compiler *c, stmt_ty s) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, (void *)type_params, loc.lineno) == -1) { Py_DECREF(type_params_name); return ERROR; @@ -2719,15 +2772,6 @@ check_compare(struct compiler *c, expr_ty e) return SUCCESS; } -static const int compare_masks[] = { - [Py_LT] = COMPARISON_LESS_THAN, - [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, - [Py_EQ] = COMPARISON_EQUALS, - [Py_NE] = COMPARISON_NOT_EQUALS, - [Py_GT] = COMPARISON_GREATER_THAN, - [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS, -}; - static int compiler_addcompare(struct compiler *c, location loc, cmpop_ty op) { @@ -6366,7 +6410,8 @@ compiler_annassign(struct compiler *c, stmt_ty s) { location loc = LOC(s); expr_ty targ = s->v.AnnAssign.target; - PyObject* mangled; + bool future_annotations = c->c_future.ff_features & CO_FUTURE_ANNOTATIONS; + PyObject *mangled; assert(s->kind == AnnAssign_kind); @@ -6384,16 +6429,30 @@ compiler_annassign(struct compiler *c, stmt_ty s) if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { - VISIT(c, annexpr, s->v.AnnAssign.annotation) + if (future_annotations) { + VISIT(c, annexpr, s->v.AnnAssign.annotation); + ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); + mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id); + ADDOP_LOAD_CONST_NEW(c, loc, mangled); + ADDOP(c, loc, STORE_SUBSCR); } else { - VISIT(c, expr, s->v.AnnAssign.annotation); + if (c->u->u_deferred_annotations == NULL) { + c->u->u_deferred_annotations = PyList_New(0); + if (c->u->u_deferred_annotations == NULL) { + return ERROR; + } + } + PyObject *ptr = PyLong_FromVoidPtr((void *)s); + if (ptr == NULL) { + return ERROR; + } + if (PyList_Append(c->u->u_deferred_annotations, ptr) < 0) { + Py_DECREF(ptr); + return ERROR; + } + Py_DECREF(ptr); } - ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); - mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id); - ADDOP_LOAD_CONST_NEW(c, loc, mangled); - ADDOP(c, loc, STORE_SUBSCR); } break; case Attribute_kind: @@ -6419,7 +6478,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) return ERROR; } /* Annotation is evaluated last. */ - if (!s->v.AnnAssign.simple && check_annotation(c, s) < 0) { + if (future_annotations && !s->v.AnnAssign.simple && check_annotation(c, s) < 0) { return ERROR; } return SUCCESS; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bab629684c53f6..470c82d938ab7c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4061,6 +4061,11 @@ assert(func_obj->func_defaults == NULL); func_obj->func_defaults = attr; break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: Py_UNREACHABLE(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 355be966cbb84a..0274f8b7a48c3c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5450,6 +5450,11 @@ assert(func_obj->func_defaults == NULL); func_obj->func_defaults = attr; break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: Py_UNREACHABLE(); } diff --git a/Python/symtable.c b/Python/symtable.c index 7e452cdb13badf..287bc2bd58107d 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -112,6 +112,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_varkeywords = 0; ste->ste_opt_lineno = 0; ste->ste_opt_col_offset = 0; + ste->ste_annotations_used = 0; ste->ste_lineno = lineno; ste->ste_col_offset = col_offset; ste->ste_end_lineno = end_lineno; @@ -132,6 +133,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_can_see_class_scope = 0; ste->ste_comp_iter_expr = 0; ste->ste_needs_classdict = 0; + ste->ste_annotation_block = NULL; ste->ste_symbols = PyDict_New(); ste->ste_varnames = PyList_New(0); @@ -167,6 +169,7 @@ ste_dealloc(PySTEntryObject *ste) Py_XDECREF(ste->ste_varnames); Py_XDECREF(ste->ste_children); Py_XDECREF(ste->ste_directives); + Py_XDECREF(ste->ste_annotation_block); Py_XDECREF(ste->ste_mangled_names); PyObject_Free(ste); } @@ -245,10 +248,11 @@ static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args); -static int symtable_visit_annotation(struct symtable *st, expr_ty annotation); +static int symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key); static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); -static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty); +static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty, + struct _symtable_entry *parent_ste); static int symtable_visit_withitem(struct symtable *st, withitem_ty item); static int symtable_visit_match_case(struct symtable *st, match_case_ty m); static int symtable_visit_pattern(struct symtable *st, pattern_ty s); @@ -504,6 +508,21 @@ _PySymtable_Lookup(struct symtable *st, void *key) return (PySTEntryObject *)v; } +int +_PySymtable_LookupOptional(struct symtable *st, void *key, + PySTEntryObject **out) +{ + PyObject *k = PyLong_FromVoidPtr(key); + if (k == NULL) { + *out = NULL; + return -1; + } + int result = PyDict_GetItemRef(st->st_blocks, k, (PyObject **)out); + Py_DECREF(k); + assert(*out == NULL || PySTEntry_Check(*out)); + return result; +} + long _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { @@ -525,6 +544,7 @@ int _PyST_IsFunctionLike(PySTEntryObject *ste) { return ste->ste_type == FunctionBlock + || ste->ste_type == AnnotationBlock || ste->ste_type == TypeVarBoundBlock || ste->ste_type == TypeAliasBlock || ste->ste_type == TypeParamBlock; @@ -1317,20 +1337,12 @@ symtable_exit_block(struct symtable *st) } static int -symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, - void *ast, int lineno, int col_offset, - int end_lineno, int end_col_offset) +symtable_enter_existing_block(struct symtable *st, PySTEntryObject* ste) { - PySTEntryObject *prev = NULL, *ste; - - ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset); - if (ste == NULL) - return 0; if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) { - Py_DECREF(ste); return 0; } - prev = st->st_cur; + PySTEntryObject *prev = st->st_cur; /* bpo-37757: For now, disallow *all* assignment expressions in the * outermost iterator expression of a comprehension, even those inside * a nested comprehension or a lambda expression. @@ -1340,21 +1352,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, } /* No need to inherit ste_mangled_names in classes, where all names * are mangled. */ - if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) { + if (prev && prev->ste_mangled_names != NULL && ste->ste_type != ClassBlock) { ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names); } /* The entry is owned by the stack. Borrow it for st_cur. */ - Py_DECREF(ste); st->st_cur = ste; - /* Annotation blocks shouldn't have any affect on the symbol table since in - * the compilation stage, they will all be transformed to strings. They are - * only created if future 'annotations' feature is activated. */ - if (block == AnnotationBlock) { + /* If "from __future__ import annotations" is active, + * annotation blocks shouldn't have any affect on the symbol table since in + * the compilation stage, they will all be transformed to strings. */ + if (st->st_future->ff_features & CO_FUTURE_ANNOTATIONS && ste->ste_type == AnnotationBlock) { return 1; } - if (block == ModuleBlock) + if (ste->ste_type == ModuleBlock) st->st_global = st->st_cur->ste_symbols; if (prev) { @@ -1365,6 +1376,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, return 1; } +static int +symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, + void *ast, int lineno, int col_offset, + int end_lineno, int end_col_offset) +{ + PySTEntryObject *ste = ste_new(st, name, block, ast, + lineno, col_offset, end_lineno, end_col_offset); + if (ste == NULL) + return 0; + int result = symtable_enter_existing_block(st, ste); + Py_DECREF(ste); + return result; +} + static long symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { @@ -1643,7 +1668,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } switch (s->kind) { - case FunctionDef_kind: + case FunctionDef_kind: { if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s))) VISIT_QUIT(st, 0); if (s->v.FunctionDef.args->defaults) @@ -1665,13 +1690,22 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params); } + PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, + LOCATION(s)); + if (!new_ste) { + VISIT_QUIT(st, 0); + } + if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args, - s->v.FunctionDef.returns)) + s->v.FunctionDef.returns, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); - if (!symtable_enter_block(st, s->v.FunctionDef.name, - FunctionBlock, (void *)s, - LOCATION(s))) + } + if (!symtable_enter_existing_block(st, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); + } + Py_DECREF(new_ste); VISIT(st, arguments, s->v.FunctionDef.args); VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st)) @@ -1681,6 +1715,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } break; + } case ClassDef_kind: { PyObject *tmp; if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s))) @@ -1776,6 +1811,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT(st, expr, s->v.Assign.value); break; case AnnAssign_kind: + st->st_cur->ste_annotations_used = 1; if (s->v.AnnAssign.target->kind == Name_kind) { expr_ty e_name = s->v.AnnAssign.target; long cur = symtable_lookup(st, e_name->v.Name.id); @@ -1810,7 +1846,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) else { VISIT(st, expr, s->v.AnnAssign.target); } - if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) { + if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation, + (void *)((uintptr_t)st->st_cur->ste_id + 1))) { VISIT_QUIT(st, 0); } @@ -1960,7 +1997,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, withitem, s->v.With.items); VISIT_SEQ(st, stmt, s->v.With.body); break; - case AsyncFunctionDef_kind: + case AsyncFunctionDef_kind: { if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s))) VISIT_QUIT(st, 0); if (s->v.AsyncFunctionDef.args->defaults) @@ -1983,14 +2020,21 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params); } + PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, + LOCATION(s)); + if (!new_ste) { + VISIT_QUIT(st, 0); + } + if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, - s->v.AsyncFunctionDef.returns)) + s->v.AsyncFunctionDef.returns, new_ste)) VISIT_QUIT(st, 0); - if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name, - FunctionBlock, (void *)s, - s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) + if (!symtable_enter_existing_block(st, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); + } + Py_DECREF(new_ste); + st->st_cur->ste_coroutine = 1; VISIT(st, arguments, s->v.AsyncFunctionDef.args); VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); @@ -2001,6 +2045,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } break; + } case AsyncWith_kind: VISIT_SEQ(st, withitem, s->v.AsyncWith.items); VISIT_SEQ(st, stmt, s->v.AsyncWith.body); @@ -2444,18 +2489,44 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args) } static int -symtable_visit_annotation(struct symtable *st, expr_ty annotation) +symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) { - int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; - if (future_annotations && - !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, - (void *)annotation, annotation->lineno, - annotation->col_offset, annotation->end_lineno, - annotation->end_col_offset)) { - VISIT_QUIT(st, 0); + struct _symtable_entry *parent_ste = st->st_cur; + if (parent_ste->ste_annotation_block == NULL) { + _Py_block_ty current_type = parent_ste->ste_type; + if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock, + key, LOCATION(annotation))) { + VISIT_QUIT(st, 0); + } + parent_ste->ste_annotation_block = + (struct _symtable_entry *)Py_NewRef(st->st_cur); + int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; + if (current_type == ClassBlock && !future_annotations) { + st->st_cur->ste_can_see_class_scope = 1; + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) { + return 0; + } + } + + _Py_DECLARE_STR(format, ".format"); + // The generated __annotate__ function takes a single parameter with the + // internal name ".format". + if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, + LOCATION(annotation))) { + return 0; + } + if (!symtable_add_def(st, &_Py_STR(format), USE, + LOCATION(annotation))) { + return 0; + } + } + else { + if (!symtable_enter_existing_block(st, parent_ste->ste_annotation_block)) { + VISIT_QUIT(st, 0); + } } VISIT(st, expr, annotation); - if (future_annotations && !symtable_exit_block(st)) { + if (!symtable_exit_block(st)) { VISIT_QUIT(st, 0); } return 1; @@ -2471,37 +2542,58 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args) for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); - if (arg->annotation) + if (arg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, arg->annotation); + } } return 1; } static int -symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns) +symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns, + struct _symtable_entry *function_ste) { - int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; - if (future_annotations && - !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, - (void *)o, o->lineno, o->col_offset, o->end_lineno, - o->end_col_offset)) { + int is_in_class = st->st_cur->ste_can_see_class_scope; + _Py_block_ty current_type = st->st_cur->ste_type; + if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock, + (void *)a, LOCATION(o))) { VISIT_QUIT(st, 0); } + if (is_in_class || current_type == ClassBlock) { + st->st_cur->ste_can_see_class_scope = 1; + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(o))) { + return 0; + } + } + _Py_DECLARE_STR(format, ".format"); + // We need to insert code that reads this "parameter" to the function. + if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, LOCATION(o))) { + return 0; + } + if (!symtable_add_def(st, &_Py_STR(format), USE, LOCATION(o))) { + return 0; + } if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs)) return 0; if (a->args && !symtable_visit_argannotations(st, a->args)) return 0; - if (a->vararg && a->vararg->annotation) + if (a->vararg && a->vararg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, a->vararg->annotation); - if (a->kwarg && a->kwarg->annotation) + } + if (a->kwarg && a->kwarg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, a->kwarg->annotation); + } if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs)) return 0; - if (future_annotations && !symtable_exit_block(st)) { - VISIT_QUIT(st, 0); + if (returns) { + st->st_cur->ste_annotations_used = 1; + VISIT(st, expr, returns); } - if (returns && !symtable_visit_annotation(st, returns)) { + if (!symtable_exit_block(st)) { VISIT_QUIT(st, 0); } return 1; @@ -2733,7 +2825,7 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e) static int symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e) { - enum _block_type type = st->st_cur->ste_type; + _Py_block_ty type = st->st_cur->ste_type; if (type == AnnotationBlock) PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name); else if (type == TypeVarBoundBlock) From ec3af291fe2f680ab277edde7113e2762754f4aa Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 11 Jun 2024 17:15:01 +0100 Subject: [PATCH 441/903] gh-120346: Respect PYTHON_BASIC_REPL when running in interactive inspect mode (#120349) --- .../2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst | 2 ++ Modules/main.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst new file mode 100644 index 00000000000000..eb2d0f9a705caa --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst @@ -0,0 +1,2 @@ +Respect :envvar:`PYTHON_BASIC_REPL` when running in interative inspect mode +(``python -i``). Patch by Pablo Galindo diff --git a/Modules/main.c b/Modules/main.c index 8eded2639ad90a..1a70b300b6ad17 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -542,7 +542,8 @@ pymain_repl(PyConfig *config, int *exitcode) return; } - if (!isatty(fileno(stdin))) { + if (!isatty(fileno(stdin)) + || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { PyCompilerFlags cf = _PyCompilerFlags_INIT; int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); *exitcode = (run != 0); From 32a0faba439b239d7b0c242c1e3cd2025c52b8cf Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 11 Jun 2024 12:42:10 -0400 Subject: [PATCH 442/903] gh-119517: Fixes for pasting in pyrepl (#120253) * Remove pyrepl's optimization for self-insert This will be replaced by a less specialized optimization. * Use line-buffering when pyrepl echoes pastes Previously echoing was totally suppressed until the entire command had been pasted and the terminal ended paste mode, but this gives the user no feedback to indicate that an operation is in progress. Drawing something to the screen once per line strikes a balance between perceived responsiveness and performance. * Remove dead code from pyrepl `msg_at_bottom` is always true. * Speed up pyrepl's screen rendering computation The Reader in pyrepl doesn't hold a complete representation of the screen area being drawn as persistent state. Instead, it recomputes it, on each keypress. This is fast enough for a few hundred bytes, but incredibly slow as the input buffer grows into the kilobytes (likely because of pasting). Rather than making some expensive and expansive changes to the repl's internal representation of the screen, add some caching: remember some data from one refresh to the next about what was drawn to the screen and, if we don't find anything that has invalidated the results that were computed last time around, reuse them. To keep this caching as simple as possible, all we'll do is look for lines in the buffer that were above the cursor the last time we were asked to update the screen, and that are still above the cursor now. We assume that nothing can affect a line that comes before both the old and new cursor location without us being informed. Based on this assumption, we can reuse old lines, which drastically speeds up the overwhelmingly common case where the user is typing near the end of the buffer. * Speed up pyrepl prompt drawing Cache the `can_colorize()` call rather than repeatedly recomputing it. This call looks up an environment variable, and is called once per character typed at the REPL. The environment variable lookup shows up as a hot spot when profiling, and we don't expect this to change while the REPL is running. * Speed up pasting multiple lines into the REPL Previously, we were checking whether the command should be accepted each time a line break was encountered, but that's not the expected behavior. In bracketed paste mode, we expect everything pasted to be part of a single block of code, and encountering a newline shouldn't behave like a user pressing to execute a command. The user should always have a chance to review the pasted command before running it. * Use a read buffer for input in pyrepl Previously we were reading one byte at a time, which causes much slower IO than necessary. Instead, read in chunks, processing previously read data before asking for more. * Optimize finding width of a single character `wlen` finds the width of a multi-character string by adding up the width of each character, and then subtracting the width of any escape sequences. It's often called for single character strings, however, which can't possibly contain escape sequences. Optimize for that case. * Optimize disp_str for ASCII characters Since every ASCII character is known to display as single width, we can avoid not only the Unicode data lookup in `disp_str` but also the one hidden in `str_width` for them. * Speed up cursor movements in long pyrepl commands When the current pyrepl command buffer contains many lines, scrolling up becomes slow. We have optimizations in place to reuse lines above the cursor position from one refresh to the next, but don't currently try to reuse lines below the cursor position in the same way, so we wind up with quadratic behavior where all lines of the buffer below the cursor are recomputed each time the cursor moves up another line. Optimize this by only computing one screen's worth of lines beyond the cursor position. Any lines beyond that can't possibly be shown by the console, and bounding this makes scrolling up have linear time complexity instead. --------- Signed-off-by: Matt Wozniski Co-authored-by: Pablo Galindo --- Lib/_pyrepl/commands.py | 3 - Lib/_pyrepl/completing_reader.py | 8 +- Lib/_pyrepl/reader.py | 154 ++++++++++++++++++++++--------- Lib/_pyrepl/readline.py | 4 + Lib/_pyrepl/unix_console.py | 20 +++- Lib/_pyrepl/utils.py | 2 + 6 files changed, 134 insertions(+), 57 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 6bffed1bfe9327..c3fce91013b001 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -368,8 +368,6 @@ def do(self) -> None: r = self.reader text = self.event * r.get_arg() r.insert(text) - if len(text) == 1 and r.pos == len(r.buffer): - r.calc_screen = r.append_to_screen class insert_nl(EditCommand): @@ -483,4 +481,3 @@ def do(self) -> None: self.reader.paste_mode = False self.reader.in_bracketed_paste = False self.reader.dirty = True - self.reader.calc_screen = self.reader.calc_complete_screen diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 8df35ccb9117b1..05770aaf5060cc 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -209,10 +209,6 @@ def do(self) -> None: r = self.reader # type: ignore[assignment] commands.self_insert.do(self) - - if r.cmpltn_menu_visible or r.cmpltn_message_visible: - r.calc_screen = r.calc_complete_screen - if r.cmpltn_menu_visible: stem = r.get_stem() if len(stem) < 1: @@ -261,8 +257,8 @@ def after_command(self, cmd: Command) -> None: if not isinstance(cmd, (complete, self_insert)): self.cmpltn_reset() - def calc_complete_screen(self) -> list[str]: - screen = super().calc_complete_screen() + def calc_screen(self) -> list[str]: + screen = super().calc_screen() if self.cmpltn_menu_visible: ly = self.lxy[1] screen[ly:ly] = self.cmpltn_menu diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index beee7764e0eb84..63ae661968408e 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -35,15 +35,13 @@ # types Command = commands.Command if False: - from typing import Callable from .types import Callback, SimpleContextManager, KeySpec, CommandName - CalcScreen = Callable[[], list[str]] def disp_str(buffer: str) -> tuple[str, list[int]]: """disp_str(buffer:string) -> (string, [int]) - Return the string that should be the printed represenation of + Return the string that should be the printed representation of |buffer| and a list detailing where the characters of |buffer| get used up. E.g.: @@ -54,11 +52,17 @@ def disp_str(buffer: str) -> tuple[str, list[int]]: b: list[int] = [] s: list[str] = [] for c in buffer: - if ord(c) > 128 and unicodedata.category(c).startswith("C"): + if ord(c) < 128: + s.append(c) + b.append(1) + elif unicodedata.category(c).startswith("C"): c = r"\u%04x" % ord(c) - s.append(c) - b.append(wlen(c)) - b.extend([0] * (len(c) - 1)) + s.append(c) + b.append(str_width(c)) + b.extend([0] * (len(c) - 1)) + else: + s.append(c) + b.append(str_width(c)) return "".join(s), b @@ -230,7 +234,6 @@ class Reader: commands: dict[str, type[Command]] = field(default_factory=make_default_commands) last_command: type[Command] | None = None syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) - msg_at_bottom: bool = True keymap: tuple[tuple[str, str], ...] = () input_trans: input.KeymapTranslator = field(init=False) input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) @@ -238,8 +241,52 @@ class Reader: screeninfo: list[tuple[int, list[int]]] = field(init=False) cxy: tuple[int, int] = field(init=False) lxy: tuple[int, int] = field(init=False) - calc_screen: CalcScreen = field(init=False) scheduled_commands: list[str] = field(default_factory=list) + can_colorize: bool = False + + ## cached metadata to speed up screen refreshes + @dataclass + class RefreshCache: + in_bracketed_paste: bool = False + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + line_end_offsets: list[int] = field(default_factory=list) + pos: int = field(init=False) + cxy: tuple[int, int] = field(init=False) + dimensions: tuple[int, int] = field(init=False) + + def update_cache(self, + reader: Reader, + screen: list[str], + screeninfo: list[tuple[int, list[int]]], + ) -> None: + self.in_bracketed_paste = reader.in_bracketed_paste + self.screen = screen.copy() + self.screeninfo = screeninfo.copy() + self.pos = reader.pos + self.cxy = reader.cxy + self.dimensions = reader.console.width, reader.console.height + + def valid(self, reader: Reader) -> bool: + dimensions = reader.console.width, reader.console.height + dimensions_changed = dimensions != self.dimensions + paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste + return not (dimensions_changed or paste_changed) + + def get_cached_location(self, reader: Reader) -> tuple[int, int]: + offset = 0 + earliest_common_pos = min(reader.pos, self.pos) + num_common_lines = len(self.line_end_offsets) + while num_common_lines > 0: + offset = self.line_end_offsets[num_common_lines - 1] + if earliest_common_pos > offset: + break + num_common_lines -= 1 + else: + offset = 0 + return offset, num_common_lines + + last_refresh_cache: RefreshCache = field(default_factory=RefreshCache) def __post_init__(self) -> None: # Enable the use of `insert` without a `prepare` call - necessary to @@ -252,53 +299,60 @@ def __post_init__(self) -> None: self.screeninfo = [(0, [])] self.cxy = self.pos2xy() self.lxy = (self.pos, 0) - self.calc_screen = self.calc_complete_screen + self.can_colorize = can_colorize() + + self.last_refresh_cache.screeninfo = self.screeninfo + self.last_refresh_cache.pos = self.pos + self.last_refresh_cache.cxy = self.cxy + self.last_refresh_cache.dimensions = (0, 0) def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: return default_keymap - def append_to_screen(self) -> list[str]: - new_screen = self.screen.copy() or [''] + def calc_screen(self) -> list[str]: + """Translate changes in self.buffer into changes in self.console.screen.""" + # Since the last call to calc_screen: + # screen and screeninfo may differ due to a completion menu being shown + # pos and cxy may differ due to edits, cursor movements, or completion menus - new_character = self.buffer[-1] - new_character_len = wlen(new_character) + # Lines that are above both the old and new cursor position can't have changed, + # unless the terminal has been resized (which might cause reflowing) or we've + # entered or left paste mode (which changes prompts, causing reflowing). + num_common_lines = 0 + offset = 0 + if self.last_refresh_cache.valid(self): + offset, num_common_lines = self.last_refresh_cache.get_cached_location(self) - last_line_len = wlen(new_screen[-1]) - if last_line_len + new_character_len >= self.console.width: # We need to wrap here - new_screen[-1] += '\\' - self.screeninfo[-1][1].append(1) - new_screen.append(self.buffer[-1]) - self.screeninfo.append((0, [new_character_len])) - else: - new_screen[-1] += self.buffer[-1] - self.screeninfo[-1][1].append(new_character_len) - self.cxy = self.pos2xy() + screen = self.last_refresh_cache.screen + del screen[num_common_lines:] - # Reset the function that is used for completing the screen - self.calc_screen = self.calc_complete_screen - return new_screen + screeninfo = self.last_refresh_cache.screeninfo + del screeninfo[num_common_lines:] + + last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets + del last_refresh_line_end_offsets[num_common_lines:] - def calc_complete_screen(self) -> list[str]: - """The purpose of this method is to translate changes in - self.buffer into changes in self.screen. Currently it rips - everything down and starts from scratch, which whilst not - especially efficient is certainly simple(r). - """ - lines = self.get_unicode().split("\n") - screen: list[str] = [] - screeninfo: list[tuple[int, list[int]]] = [] pos = self.pos - for ln, line in enumerate(lines): + pos -= offset + + lines = "".join(self.buffer[offset:]).split("\n") + cursor_found = False + lines_beyond_cursor = 0 + for ln, line in enumerate(lines, num_common_lines): ll = len(line) if 0 <= pos <= ll: - if self.msg and not self.msg_at_bottom: - for mline in self.msg.split("\n"): - screen.append(mline) - screeninfo.append((0, [])) self.lxy = pos, ln + cursor_found = True + elif cursor_found: + lines_beyond_cursor += 1 + if lines_beyond_cursor > self.console.height: + # No need to keep formatting lines. + # The console can't show them. + break prompt = self.get_prompt(ln, ll >= pos >= 0) while "\n" in prompt: pre_prompt, _, prompt = prompt.partition("\n") + last_refresh_line_end_offsets.append(offset) screen.append(pre_prompt) screeninfo.append((0, [])) pos -= ll + 1 @@ -306,6 +360,8 @@ def calc_complete_screen(self) -> list[str]: l, l2 = disp_str(line) wrapcount = (wlen(l) + lp) // self.console.width if wrapcount == 0: + offset += ll + 1 # Takes all of the line plus the newline + last_refresh_line_end_offsets.append(offset) screen.append(prompt + l) screeninfo.append((lp, l2)) else: @@ -321,11 +377,14 @@ def calc_complete_screen(self) -> list[str]: column += character_width pre = prompt if i == 0 else "" if len(l) > index_to_wrap_before: + offset += index_to_wrap_before post = "\\" after = [1] else: + offset += index_to_wrap_before + 1 # Takes the newline post = "" after = [] + last_refresh_line_end_offsets.append(offset) screen.append(pre + l[:index_to_wrap_before] + post) screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) l = l[index_to_wrap_before:] @@ -333,10 +392,12 @@ def calc_complete_screen(self) -> list[str]: i += 1 self.screeninfo = screeninfo self.cxy = self.pos2xy() - if self.msg and self.msg_at_bottom: + if self.msg: for mline in self.msg.split("\n"): screen.append(mline) screeninfo.append((0, [])) + + self.last_refresh_cache.update_cache(self, screen, screeninfo) return screen @staticmethod @@ -456,7 +517,7 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: 'lineno'.""" if self.arg is not None and cursor_on_line: prompt = f"(arg: {self.arg}) " - elif self.paste_mode: + elif self.paste_mode and not self.in_bracketed_paste: prompt = "(paste) " elif "\n" in self.buffer: if lineno == 0: @@ -468,7 +529,7 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: else: prompt = self.ps1 - if can_colorize(): + if self.can_colorize: prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}" return prompt @@ -604,6 +665,9 @@ def update_screen(self) -> None: def refresh(self) -> None: """Recalculate and refresh the screen.""" + if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n": + return + # this call sets up self.cxy, so call it first. self.screen = self.calc_screen() self.console.refresh(self.screen, self.cxy) @@ -627,7 +691,7 @@ def do_cmd(self, cmd: tuple[str, list[str]]) -> None: self.after_command(command) - if self.dirty and not self.in_bracketed_paste: + if self.dirty: self.refresh() else: self.update_cursor() diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 7d811bf41773fe..b10d0c66e4f813 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -263,6 +263,10 @@ def do(self) -> None: r = self.reader # type: ignore[assignment] r.dirty = True # this is needed to hide the completion menu, if visible + if self.reader.in_bracketed_paste: + r.insert("\n") + return + # if there are already several lines and the cursor # is not on the last one, always insert a new \n. text = r.get_unicode() diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 2f73a59dd1fced..f1a6b84adfb671 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -150,6 +150,8 @@ def __init__( self.pollob = poll() self.pollob.register(self.input_fd, select.POLLIN) + self.input_buffer = b"" + self.input_buffer_pos = 0 curses.setupterm(term or None, self.output_fd) self.term = term @@ -197,6 +199,18 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None: self.event_queue = EventQueue(self.input_fd, self.encoding) self.cursor_visible = 1 + def __read(self, n: int) -> bytes: + if not self.input_buffer or self.input_buffer_pos >= len(self.input_buffer): + self.input_buffer = os.read(self.input_fd, 10000) + + ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n] + self.input_buffer_pos += len(ret) + if self.input_buffer_pos >= len(self.input_buffer): + self.input_buffer = b"" + self.input_buffer_pos = 0 + return ret + + def change_encoding(self, encoding: str) -> None: """ Change the encoding used for I/O operations. @@ -373,7 +387,7 @@ def get_event(self, block: bool = True) -> Event | None: while self.event_queue.empty(): while True: try: - self.push_char(os.read(self.input_fd, 1)) + self.push_char(self.__read(1)) except OSError as err: if err.errno == errno.EINTR: if not self.event_queue.empty(): @@ -491,7 +505,7 @@ def getpending(self): e.raw += e.raw amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] - raw = os.read(self.input_fd, amount) + raw = self.__read(amount) data = str(raw, self.encoding, "replace") e.data += data e.raw += raw @@ -514,7 +528,7 @@ def getpending(self): e.raw += e.raw amount = 10000 - raw = os.read(self.input_fd, amount) + raw = self.__read(amount) data = str(raw, self.encoding, "replace") e.data += data e.raw += raw diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 96e917e487d91a..20dbb1f7e17229 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -16,6 +16,8 @@ def str_width(c: str) -> int: def wlen(s: str) -> int: + if len(s) == 1: + return str_width(s) length = sum(str_width(i) for i in s) # remove lengths of any escape sequences sequence = ANSI_ESCAPE_SEQUENCE.findall(s) From 1b62bcee941e54244b3ce6476aef8913604987c9 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 11 Jun 2024 19:00:53 +0200 Subject: [PATCH 443/903] gh-120343: Do not reset byte_col_offset_diff after multiline tokens (#120352) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/test/test_tokenize.py | 11 +++++++++++ .../2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst | 1 + Python/Python-tokenize.c | 7 ++++++- 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 4428e8cea1964c..36dba71766cc20 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1199,6 +1199,17 @@ def test_closing_parenthesis_from_different_line(self): NAME 'x' (1, 3) (1, 4) """) + def test_multiline_non_ascii_fstring(self): + self.check_tokenize("""\ +a = f''' + Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli'''""", """\ + NAME 'a' (1, 0) (1, 1) + OP '=' (1, 2) (1, 3) + FSTRING_START "f\'\'\'" (1, 4) (1, 8) + FSTRING_MIDDLE '\\n Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68) + FSTRING_END "\'\'\'" (2, 68) (2, 71) + """) + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. diff --git a/Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst b/Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst new file mode 100644 index 00000000000000..76714b0c394eef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-11-16-34-41.gh-issue-120343.hdiXeU.rst @@ -0,0 +1 @@ +Fix column offset reporting for tokens that come after multiline f-strings in the :mod:`tokenize` module. diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 09fad18b5b4df7..2591dae35736ba 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -36,6 +36,7 @@ typedef struct /* Needed to cache line for performance */ PyObject *last_line; Py_ssize_t last_lineno; + Py_ssize_t last_end_lineno; Py_ssize_t byte_col_offset_diff; } tokenizeriterobject; @@ -77,6 +78,7 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, self->last_line = NULL; self->byte_col_offset_diff = 0; self->last_lineno = 0; + self->last_end_lineno = 0; return (PyObject *)self; } @@ -227,7 +229,9 @@ tokenizeriter_next(tokenizeriterobject *it) Py_XDECREF(it->last_line); line = PyUnicode_DecodeUTF8(line_start, size, "replace"); it->last_line = line; - it->byte_col_offset_diff = 0; + if (it->tok->lineno != it->last_end_lineno) { + it->byte_col_offset_diff = 0; + } } else { // Line hasn't changed so we reuse the cached one. line = it->last_line; @@ -241,6 +245,7 @@ tokenizeriter_next(tokenizeriterobject *it) Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno; Py_ssize_t end_lineno = it->tok->lineno; it->last_lineno = lineno; + it->last_end_lineno = end_lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; From 0335662fe1f663fe96e3e4acf0f34c5959d06b00 Mon Sep 17 00:00:00 2001 From: naglis <827324+naglis@users.noreply.github.com> Date: Tue, 11 Jun 2024 20:01:48 +0300 Subject: [PATCH 444/903] Fix typo in ElementTree docs (#120342) --- Doc/library/xml.etree.elementtree.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index e5919029c62c93..4c1e7bd7e6734a 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -508,7 +508,7 @@ Functions `C14N 2.0 `_ transformation function. Canonicalization is a way to normalise XML output in a way that allows - byte-by-byte comparisons and digital signatures. It reduced the freedom + byte-by-byte comparisons and digital signatures. It reduces the freedom that XML serializers have and instead generates a more constrained XML representation. The main restrictions regard the placement of namespace declarations, the ordering of attributes, and ignorable whitespace. From 86a8a1c57a386fb3330bee0fa44fc3fd6c3042a3 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Tue, 11 Jun 2024 13:40:31 -0400 Subject: [PATCH 445/903] gh-118908: Limit exposed globals from internal imports and definitions on new REPL startup (#119547) --- Lib/_pyrepl/simple_interact.py | 21 ++++++- Lib/test/test_pyrepl/test_pyrepl.py | 63 ++++++++++++++++++- Lib/test/test_repl.py | 5 +- ...-05-25-10-40-38.gh-issue-118908.XcZiq4.rst | 2 + 4 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 2e5698eb131684..620f87b4867073 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -27,6 +27,7 @@ import _sitebuiltins import linecache +import builtins import sys import code from types import ModuleType @@ -34,6 +35,12 @@ from .console import InteractiveColoredConsole from .readline import _get_reader, multiline_input +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + + _error: tuple[type[Exception], ...] | type[Exception] try: from .unix_console import _error @@ -73,20 +80,28 @@ def _clear_screen(): "clear": _clear_screen, } +DEFAULT_NAMESPACE: dict[str, Any] = { + '__name__': '__main__', + '__doc__': None, + '__package__': None, + '__loader__': None, + '__spec__': None, + '__annotations__': {}, + '__builtins__': builtins, +} def run_multiline_interactive_console( mainmodule: ModuleType | None = None, future_flags: int = 0, console: code.InteractiveConsole | None = None, ) -> None: - import __main__ from .readline import _setup _setup() - mainmodule = mainmodule or __main__ + namespace = mainmodule.__dict__ if mainmodule else DEFAULT_NAMESPACE if console is None: console = InteractiveColoredConsole( - mainmodule.__dict__, filename="" + namespace, filename="" ) if future_flags: console.compile.compiler.flags |= future_flags diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 45114e7315749f..3167b8473bfe20 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1,9 +1,13 @@ -import itertools import io +import itertools import os import rlcompleter -from unittest import TestCase +import select +import subprocess +import sys +from unittest import TestCase, skipUnless from unittest.mock import patch +from test.support import force_not_colorized from .support import ( FakeConsole, @@ -17,6 +21,10 @@ from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig from _pyrepl.readline import multiline_input as readline_multiline_input +try: + import pty +except ImportError: + pty = None class TestCursorPosition(TestCase): def prepare_reader(self, events): @@ -828,3 +836,54 @@ def test_bracketed_paste_single_line(self): reader = self.prepare_reader(events) output = multiline_input(reader) self.assertEqual(output, input_code) + + +@skipUnless(pty, "requires pty") +class TestMain(TestCase): + @force_not_colorized + def test_exposed_globals_in_repl(self): + expected_output = ( + "[\'__annotations__\', \'__builtins__\', \'__doc__\', \'__loader__\', " + "\'__name__\', \'__package__\', \'__spec__\']" + ) + output, exit_code = self.run_repl(["sorted(dir())", "exit"]) + if "can\'t use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertEqual(exit_code, 0) + self.assertIn(expected_output, output) + + def test_dumb_terminal_exits_cleanly(self): + env = os.environ.copy() + env.update({"TERM": "dumb"}) + output, exit_code = self.run_repl("exit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("warning: can\'t use pyrepl", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + def run_repl(self, repl_input: str | list[str], env: dict | None = None) -> tuple[str, int]: + master_fd, slave_fd = pty.openpty() + process = subprocess.Popen( + [sys.executable, "-i", "-u"], + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + text=True, + close_fds=True, + env=env if env else os.environ, + ) + if isinstance(repl_input, list): + repl_input = "\n".join(repl_input) + "\n" + os.write(master_fd, repl_input.encode("utf-8")) + + output = [] + while select.select([master_fd], [], [], 0.5)[0]: + data = os.read(master_fd, 1024).decode("utf-8") + if not data: + break + output.append(data) + + os.close(master_fd) + os.close(slave_fd) + exit_code = process.wait() + return "\n".join(output), exit_code diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 340178366fc13a..1caf09ceaf10fc 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -1,9 +1,9 @@ """Test the interactive interpreter.""" -import sys import os -import unittest import subprocess +import sys +import unittest from textwrap import dedent from test import support from test.support import cpython_only, has_subprocess_support, SuppressCrashReport @@ -199,7 +199,6 @@ def test_asyncio_repl_is_ok(self): assert_python_ok("-m", "asyncio") - class TestInteractiveModeSyntaxErrors(unittest.TestCase): def test_interactive_syntax_error_correct_line(self): diff --git a/Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst b/Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst new file mode 100644 index 00000000000000..bf58d7277fcd51 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-10-40-38.gh-issue-118908.XcZiq4.rst @@ -0,0 +1,2 @@ +Limit exposed globals from internal imports and definitions on new REPL +startup. Patch by Eugene Triguba and Pablo Galindo. From 939c201e00943c6dc2d515185168c30606ae522c Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Tue, 11 Jun 2024 20:50:21 +0300 Subject: [PATCH 446/903] gh-120326: Include on Windows with Free Threading (#120329) --- Include/Python.h | 4 ++++ .../next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst diff --git a/Include/Python.h b/Include/Python.h index 502c5ec5aeaa3c..a1b33f6d3c42b2 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -51,6 +51,10 @@ # error "The limited API is not currently supported in the free-threaded build" #endif +#if defined(Py_GIL_DISABLED) && defined(_MSC_VER) +# include // __readgsqword() +#endif + // Include Python header files #include "pyport.h" #include "pymacro.h" diff --git a/Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst b/Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst new file mode 100644 index 00000000000000..25cbdf6ba50ab8 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-11-00-38-05.gh-issue-120326.JHSDF1.rst @@ -0,0 +1,2 @@ +On Windows, fix build error when ``--disable-gil`` and ``--experimental-jit`` +options are combined. From 203565b2f9c74656ba519780049b46d4e5afcba1 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Wed, 12 Jun 2024 03:10:23 +0800 Subject: [PATCH 447/903] gh-120198: Fix race condition when editing __class__ with an audit hook active (GH-120195) --- Lib/test/test_free_threading/test_type.py | 1 + Lib/test/test_super.py | 35 ++++++++++++++++++- ...-06-10-15-07-16.gh-issue-120198.WW_pjO.rst | 1 + Objects/typeobject.c | 3 +- 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 6eead198deed46..786336fa0cddce 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -1,3 +1,4 @@ +import threading import unittest from concurrent.futures import ThreadPoolExecutor diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 256b416caaa584..3ffbe03f0c2f11 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -1,9 +1,10 @@ """Unit tests for zero-argument super() & related machinery.""" import textwrap +import threading import unittest from unittest.mock import patch -from test.support import import_helper +from test.support import import_helper, threading_helper ADAPTIVE_WARMUP_DELAY = 2 @@ -505,6 +506,38 @@ def some(cls): for _ in range(ADAPTIVE_WARMUP_DELAY): C.some(C) + @threading_helper.requires_working_threading() + def test___class___modification_multithreaded(self): + """ Note: this test isn't actually testing anything on its own. + It requires a sys audithook to be set to crash on older Python. + This should be the case anyways as our test suite sets + an audit hook. + """ + class Foo: + pass + + class Bar: + pass + + thing = Foo() + def work(): + foo = thing + for _ in range(5000): + foo.__class__ = Bar + type(foo) + foo.__class__ = Foo + type(foo) + + + threads = [] + for _ in range(6): + thread = threading.Thread(target=work) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst new file mode 100644 index 00000000000000..8dc8aec44d80c4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-15-07-16.gh-issue-120198.WW_pjO.rst @@ -0,0 +1 @@ +Fix a crash when multiple threads read and write to the same ``__class__`` of an object concurrently. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index cd16bebd1e1cb8..070e3d2f7bf2b4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6522,7 +6522,6 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* static int object_set_class(PyObject *self, PyObject *value, void *closure) { - PyTypeObject *oldto = Py_TYPE(self); if (value == NULL) { PyErr_SetString(PyExc_TypeError, @@ -6542,6 +6541,8 @@ object_set_class(PyObject *self, PyObject *value, void *closure) return -1; } + PyTypeObject *oldto = Py_TYPE(self); + /* In versions of CPython prior to 3.5, the code in compatible_for_assignment was not set up to correctly check for memory layout / slot / etc. compatibility for non-HEAPTYPE classes, so we just From 34e4d3287e724c065cc07b04a1ee8715817db284 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 11 Jun 2024 20:20:25 +0100 Subject: [PATCH 448/903] gh-120221: Deliver real singals on Ctrl-C and Ctrl-Z in the new REPL (#120354) --- Lib/_pyrepl/unix_console.py | 8 ++++---- .../2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index f1a6b84adfb671..af9290819c2c78 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -324,13 +324,13 @@ def prepare(self): """ self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() - raw.iflag &= ~(termios.BRKINT | termios.INPCK | termios.ISTRIP | termios.IXON) + raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) raw.oflag &= ~(termios.OPOST) raw.cflag &= ~(termios.CSIZE | termios.PARENB) raw.cflag |= termios.CS8 - raw.lflag &= ~( - termios.ICANON | termios.ECHO | termios.IEXTEN | (termios.ISIG * 1) - ) + raw.iflag |= termios.BRKINT + raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN) + raw.lflag |= termios.ISIG raw.cc[termios.VMIN] = 1 raw.cc[termios.VTIME] = 0 tcsetattr(self.input_fd, termios.TCSADRAIN, raw) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst new file mode 100644 index 00000000000000..3781576bc5a257 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-17-56-12.gh-issue-120221.si9hM9.rst @@ -0,0 +1,2 @@ +Deliver real signals on Ctrl-C and Ctrl-Z in the new REPL. Patch by Pablo +Galindo From f5a9c34f38886c5cf9c2f8d860eee3473447e030 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 12 Jun 2024 04:00:56 +0300 Subject: [PATCH 449/903] gh-120056: Add `IP_RECVERR`, `IP_RECVORIGDSTADDR`, `IP_RECVTTL` to `socket` module (#120058) * gh-120056: Add `IP_RECVERR` and `IP_RECVTTL` to `socket` module * Fix news * Address review * Update NEWS --- Doc/library/socket.rst | 4 ++++ .../2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst | 3 +++ Modules/socketmodule.c | 9 +++++++++ 3 files changed, 16 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 2df0257d1f24f0..782fb9b27ae1ba 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -450,6 +450,10 @@ Constants same way that ``SO_BINDTODEVICE`` is used, but with the index of a network interface instead of its name. + .. versionchanged:: 3.14 + Added missing ``IP_RECVERR``, ``IP_RECVTTL``, and ``IP_RECVORIGDSTADDR`` + on Linux. + .. data:: AF_CAN PF_CAN SOL_CAN_* diff --git a/Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst b/Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst new file mode 100644 index 00000000000000..0adb70f51e8a0c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-19-49-16.gh-issue-120056.5aqozw.rst @@ -0,0 +1,3 @@ +Add :data:`!socket.IP_RECVERR` and :data:`!socket.IP_RECVTTL` constants +(both available since Linux 2.2). +And :data:`!socket.IP_RECVORIGDSTADDR` constant (available since Linux 2.6.29). diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index cb7dc25e23fb3d..0626d7934983db 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8412,15 +8412,24 @@ socket_exec(PyObject *m) #ifdef IP_TTL ADD_INT_MACRO(m, IP_TTL); #endif +#ifdef IP_RECVERR + ADD_INT_MACRO(m, IP_RECVERR); +#endif #ifdef IP_RECVOPTS ADD_INT_MACRO(m, IP_RECVOPTS); #endif +#ifdef IP_RECVORIGDSTADDR + ADD_INT_MACRO(m, IP_RECVORIGDSTADDR); +#endif #ifdef IP_RECVRETOPTS ADD_INT_MACRO(m, IP_RECVRETOPTS); #endif #ifdef IP_RECVTOS ADD_INT_MACRO(m, IP_RECVTOS); #endif +#ifdef IP_RECVTTL + ADD_INT_MACRO(m, IP_RECVTTL); +#endif #ifdef IP_RECVDSTADDR ADD_INT_MACRO(m, IP_RECVDSTADDR); #endif From 19435d299a1fae9ad9a6bbe6609e41ddfd7f6cbe Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 12 Jun 2024 10:37:14 +0300 Subject: [PATCH 450/903] gh-120385: Fix reference leak in symtable (#120386) Decref 'new_ste' if symtable_visit_annotations() fails. --- Python/symtable.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/symtable.c b/Python/symtable.c index 287bc2bd58107d..627184da9ef4ed 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -2027,8 +2027,10 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, - s->v.AsyncFunctionDef.returns, new_ste)) + s->v.AsyncFunctionDef.returns, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); + } if (!symtable_enter_existing_block(st, new_ste)) { Py_DECREF(new_ste); VISIT_QUIT(st, 0); From 02e74c356223feb0771759286d24d1dbac01d4ca Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 12 Jun 2024 10:21:53 +0200 Subject: [PATCH 451/903] gh-118908: Fix completions after namespace change in REPL (#120370) --- Lib/_pyrepl/readline.py | 13 ++++++++++--- Lib/_pyrepl/simple_interact.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index b10d0c66e4f813..28f592d80b1b03 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -55,6 +55,11 @@ from collections.abc import Callable, Collection from .types import Callback, Completer, KeySpec, CommandName +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + MoreLinesCallable = Callable[[str], bool] @@ -92,7 +97,7 @@ @dataclass class ReadlineConfig: - readline_completer: Completer | None = RLCompleter().complete + readline_completer: Completer | None = None completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") @@ -554,7 +559,7 @@ def stub(*args: object, **kwds: object) -> None: # ____________________________________________________________ -def _setup() -> None: +def _setup(namespace: dict[str, Any]) -> None: global raw_input if raw_input is not None: return # don't run _setup twice @@ -570,9 +575,11 @@ def _setup() -> None: _wrapper.f_in = f_in _wrapper.f_out = f_out + # set up namespace in rlcompleter + _wrapper.config.readline_completer = RLCompleter(namespace).complete + # this is not really what readline.c does. Better than nothing I guess import builtins - raw_input = builtins.input builtins.input = _wrapper.input diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 620f87b4867073..2de3b38c37a9da 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -96,9 +96,9 @@ def run_multiline_interactive_console( console: code.InteractiveConsole | None = None, ) -> None: from .readline import _setup - _setup() - namespace = mainmodule.__dict__ if mainmodule else DEFAULT_NAMESPACE + _setup(namespace) + if console is None: console = InteractiveColoredConsole( namespace, filename="" From 7dd8c37a067f9fcb6a2a658d6a93b294cc2e6fb4 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:07:25 +0700 Subject: [PATCH 452/903] gh-101575: document Decimal.__round__() (GH-101737) --- Doc/library/decimal.rst | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 3e33581f96f16a..db323802a6f68c 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -897,6 +897,48 @@ Decimal objects :const:`Rounded`. If given, applies *rounding*; otherwise, uses the rounding method in either the supplied *context* or the current context. + Decimal numbers can be rounded using the :func:`.round` function: + + .. describe:: round(number) + .. describe:: round(number, ndigits) + + If *ndigits* is not given or ``None``, + returns the nearest :class:`int` to *number*, + rounding ties to even, and ignoring the rounding mode of the + :class:`Decimal` context. Raises :exc:`OverflowError` if *number* is an + infinity or :exc:`ValueError` if it is a (quiet or signaling) NaN. + + If *ndigits* is an :class:`int`, the context's rounding mode is respected + and a :class:`Decimal` representing *number* rounded to the nearest + multiple of ``Decimal('1E-ndigits')`` is returned; in this case, + ``round(number, ndigits)`` is equivalent to + ``self.quantize(Decimal('1E-ndigits'))``. Returns ``Decimal('NaN')`` if + *number* is a quiet NaN. Raises :class:`InvalidOperation` if *number* + is an infinity, a signaling NaN, or if the length of the coefficient after + the quantize operation would be greater than the current context's + precision. In other words, for the non-corner cases: + + * if *ndigits* is positive, return *number* rounded to *ndigits* decimal + places; + * if *ndigits* is zero, return *number* rounded to the nearest integer; + * if *ndigits* is negative, return *number* rounded to the nearest + multiple of ``10**abs(ndigits)``. + + For example:: + + >>> from decimal import Decimal, getcontext, ROUND_DOWN + >>> getcontext().rounding = ROUND_DOWN + >>> round(Decimal('3.75')) # context rounding ignored + 4 + >>> round(Decimal('3.5')) # round-ties-to-even + 4 + >>> round(Decimal('3.75'), 0) # uses the context rounding + Decimal('3') + >>> round(Decimal('3.75'), 1) + Decimal('3.7') + >>> round(Decimal('3.75'), -1) + Decimal('0E+1') + .. _logical_operands_label: From 755dab719dfc924dd8aef46f67512dabb8f25071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:14:50 +0200 Subject: [PATCH 453/903] gh-120029: make `symtable.Symbol.__repr__` correctly reflect the compiler's flags, add methods (#120099) Expose :class:`symtable.Symbol` methods :meth:`~symtable.Symbol.is_free_class`, :meth:`~symtable.Symbol.is_comp_iter` and :meth:`~symtable.Symbol.is_comp_cell`. --------- Co-authored-by: Carl Meyer --- Doc/library/symtable.rst | 34 +++++++++++++++++++ Doc/whatsnew/3.14.rst | 11 ++++++ Include/internal/pycore_symtable.h | 2 +- Lib/symtable.py | 32 ++++++++++++++--- Lib/test/test_symtable.py | 21 ++++++++++++ ...-06-05-11-03-10.gh-issue-120029.QBsw47.rst | 4 +++ Modules/symtablemodule.c | 2 ++ 7 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index e17a33f7feb1ab..050a941d9d0516 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -155,6 +155,8 @@ Examining Symbol Tables Return ``True`` if the symbol is a type parameter. + .. versionadded:: 3.14 + .. method:: is_global() Return ``True`` if the symbol is global. @@ -182,10 +184,42 @@ Examining Symbol Tables Return ``True`` if the symbol is referenced in its block, but not assigned to. + .. method:: is_free_class() + + Return *True* if a class-scoped symbol is free from + the perspective of a method. + + Consider the following example:: + + def f(): + x = 1 # function-scoped + class C: + x = 2 # class-scoped + def method(self): + return x + + In this example, the class-scoped symbol ``x`` is considered to + be free from the perspective of ``C.method``, thereby allowing + the latter to return *1* at runtime and not *2*. + + .. versionadded:: 3.14 + .. method:: is_assigned() Return ``True`` if the symbol is assigned to in its block. + .. method:: is_comp_iter() + + Return ``True`` if the symbol is a comprehension iteration variable. + + .. versionadded:: 3.14 + + .. method:: is_comp_cell() + + Return ``True`` if the symbol is a cell in an inlined comprehension. + + .. versionadded:: 3.14 + .. method:: is_namespace() Return ``True`` if name binding introduces new namespace. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b77ff30a8fbbee..b357553735e8bb 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -100,6 +100,17 @@ os by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) +symtable +-------- + +* Expose the following :class:`symtable.Symbol` methods: + + * :meth:`~symtable.Symbol.is_free_class` + * :meth:`~symtable.Symbol.is_comp_iter` + * :meth:`~symtable.Symbol.is_comp_cell` + + (Contributed by Bénédikt Tran in :gh:`120029`.) + Optimizations ============= diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 5d544765237df5..1be48edc80c830 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -154,7 +154,7 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) /* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol - table. GLOBAL is returned from PyST_GetScope() for either of them. + table. GLOBAL is returned from _PyST_GetScope() for either of them. It is stored in ste_symbols at bits 13-16. */ #define SCOPE_OFFSET 12 diff --git a/Lib/symtable.py b/Lib/symtable.py index af65e93e68eda4..d6ac1f527ba8ba 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -4,7 +4,10 @@ from _symtable import ( USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, - DEF_PARAM, DEF_TYPE_PARAM, DEF_IMPORT, DEF_BOUND, DEF_ANNOT, + DEF_PARAM, DEF_TYPE_PARAM, + DEF_FREE_CLASS, + DEF_IMPORT, DEF_BOUND, DEF_ANNOT, + DEF_COMP_ITER, DEF_COMP_CELL, SCOPE_OFF, SCOPE_MASK, FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL ) @@ -158,6 +161,10 @@ def get_children(self): for st in self._table.children] +def _get_scope(flags): # like _PyST_GetScope() + return (flags >> SCOPE_OFF) & SCOPE_MASK + + class Function(SymbolTable): # Default values for instance variables @@ -183,7 +190,7 @@ def get_locals(self): """ if self.__locals is None: locs = (LOCAL, CELL) - test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs + test = lambda x: _get_scope(x) in locs self.__locals = self.__idents_matching(test) return self.__locals @@ -192,7 +199,7 @@ def get_globals(self): """ if self.__globals is None: glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) - test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob + test = lambda x: _get_scope(x) in glob self.__globals = self.__idents_matching(test) return self.__globals @@ -207,7 +214,7 @@ def get_frees(self): """Return a tuple of free variables in the function. """ if self.__frees is None: - is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE + is_free = lambda x: _get_scope(x) == FREE self.__frees = self.__idents_matching(is_free) return self.__frees @@ -234,7 +241,7 @@ class Symbol: def __init__(self, name, flags, namespaces=None, *, module_scope=False): self.__name = name self.__flags = flags - self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope() + self.__scope = _get_scope(flags) self.__namespaces = namespaces or () self.__module_scope = module_scope @@ -303,6 +310,11 @@ def is_free(self): """ return bool(self.__scope == FREE) + def is_free_class(self): + """Return *True* if a class-scoped symbol is free from + the perspective of a method.""" + return bool(self.__flags & DEF_FREE_CLASS) + def is_imported(self): """Return *True* if the symbol is created from an import statement. @@ -313,6 +325,16 @@ def is_assigned(self): """Return *True* if a symbol is assigned to.""" return bool(self.__flags & DEF_LOCAL) + def is_comp_iter(self): + """Return *True* if the symbol is a comprehension iteration variable. + """ + return bool(self.__flags & DEF_COMP_ITER) + + def is_comp_cell(self): + """Return *True* if the symbol is a cell in an inlined comprehension. + """ + return bool(self.__flags & DEF_COMP_CELL) + def is_namespace(self): """Returns *True* if name binding introduces new namespace. diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index a4b111e865c86e..903c6d66f50964 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -304,6 +304,27 @@ def test_symbol_repr(self): self.assertEqual(repr(self.GenericMine.lookup("T")), "") + st1 = symtable.symtable("[x for x in [1]]", "?", "exec") + self.assertEqual(repr(st1.lookup("x")), + "") + + st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec") + self.assertEqual(repr(st2.lookup("x")), + "") + + st3 = symtable.symtable("def f():\n" + " x = 1\n" + " class A:\n" + " x = 2\n" + " def method():\n" + " return x\n", + "?", "exec") + # child 0 is for __annotate__ + func_f = st3.get_children()[1] + class_A = func_f.get_children()[0] + self.assertEqual(repr(class_A.lookup('x')), + "") + def test_symtable_entry_repr(self): expected = f"" self.assertEqual(repr(self.top._table), expected) diff --git a/Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst b/Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst new file mode 100644 index 00000000000000..d1b2c592a113ce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-05-11-03-10.gh-issue-120029.QBsw47.rst @@ -0,0 +1,4 @@ +Expose :class:`symtable.Symbol` methods :meth:`~symtable.Symbol.is_free_class`, +:meth:`~symtable.Symbol.is_comp_iter` and :meth:`~symtable.Symbol.is_comp_cell`. +Patch by Bénédikt Tran. + diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 63c4dd4225298d..b39b59bf7b06bf 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -81,6 +81,8 @@ symtable_init_constants(PyObject *m) if (PyModule_AddIntMacro(m, DEF_IMPORT) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_BOUND) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_ANNOT) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_COMP_ITER) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_COMP_CELL) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock) < 0) return -1; From 97b69db167be28a33688db436551a6c3c3ea4662 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 12 Jun 2024 12:53:19 +0100 Subject: [PATCH 454/903] gh-93691: fix too broad source locations of for statement iterators (#120330) --- Lib/test/test_compiler_codegen.py | 1 + Lib/test/test_iter.py | 46 +++++++++++++++++++ Lib/test/test_sys_settrace.py | 6 +-- ...4-06-10-22-30-26.gh-issue-93691.68WOTS.rst | 2 + Programs/test_frozenmain.h | 9 ++-- Python/compile.c | 7 +++ 6 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 1088b4aa9e624d..d82fb85ed259ab 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -49,6 +49,7 @@ def test_for_loop(self): ('GET_ITER', None, 1), loop_lbl := self.Label(), ('FOR_ITER', exit_lbl := self.Label(), 1), + ('NOP', None, 1, 1), ('STORE_NAME', 1, 1), ('LOAD_NAME', 2, 2), ('PUSH_NULL', None, 2), diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 9606d5beab71cb..ec2b68acb90785 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -10,6 +10,7 @@ import functools import contextlib import builtins +import traceback # Test result of triple loop (too big to inline) TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2), @@ -1143,6 +1144,51 @@ def test_error_iter(self): self.assertRaises(TypeError, iter, typ()) self.assertRaises(ZeroDivisionError, iter, BadIterableClass()) + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + + class Iter: + def __init__(self, init_raises=False, next_raises=False): + if init_raises: + 1/0 + self.next_raises = next_raises + + def __next__(self): + if self.next_raises: + 1/0 + + def __iter__(self): + return self + + def init_raises(): + try: + for x in Iter(init_raises=True): + pass + except Exception as e: + return e + + def next_raises(): + try: + for x in Iter(next_raises=True): + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "Iter(init_raises=True)"), + (next_raises, "Iter(next_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index ded1d9224d82d9..c622fd9ce7c466 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1650,15 +1650,15 @@ def func(): EXPECTED_EVENTS = [ (0, 'call'), (2, 'line'), - (1, 'line'), (-3, 'call'), (-2, 'line'), (-2, 'return'), - (4, 'line'), (1, 'line'), + (4, 'line'), + (2, 'line'), (-2, 'call'), (-2, 'return'), - (1, 'return'), + (2, 'return'), ] # C level events should be the same as expected and the same as Python level. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst new file mode 100644 index 00000000000000..294f8d892b459b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-22-30-26.gh-issue-93691.68WOTS.rst @@ -0,0 +1,2 @@ +Fix source locations of instructions generated for the iterator of a for +statement. diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index cdc417e48ebec6..f34d7ea0789310 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -27,12 +27,11 @@ unsigned char M_test_frozenmain[] = { 3,0,0,0,218,3,107,101,121,169,0,243,0,0,0,0, 218,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105, 110,46,112,121,218,8,60,109,111,100,117,108,101,62,114,18, - 0,0,0,1,0,0,0,115,99,0,0,0,240,3,1,1, + 0,0,0,1,0,0,0,115,94,0,0,0,240,3,1,1, 1,243,8,0,1,11,219,0,24,225,0,5,208,6,26,212, 0,27,217,0,5,128,106,144,35,151,40,145,40,212,0,27, 216,9,26,215,9,38,210,9,38,211,9,40,168,24,209,9, - 50,128,6,240,2,6,12,2,242,0,7,1,42,128,67,241, - 14,0,5,10,136,71,144,67,144,53,152,2,152,54,160,35, - 153,59,152,45,208,10,40,214,4,41,242,15,7,1,42,114, - 16,0,0,0, + 50,128,6,243,2,6,12,2,128,67,241,14,0,5,10,136, + 71,144,67,144,53,152,2,152,54,160,35,153,59,152,45,208, + 10,40,214,4,41,242,15,6,12,2,114,16,0,0,0, }; diff --git a/Python/compile.c b/Python/compile.c index c3372766d0bd50..749b69f5911386 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3025,11 +3025,18 @@ compiler_for(struct compiler *c, stmt_ty s) RETURN_IF_ERROR(compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)); VISIT(c, expr, s->v.For.iter); + + loc = LOC(s->v.For.iter); ADDOP(c, loc, GET_ITER); USE_LABEL(c, start); ADDOP_JUMP(c, loc, FOR_ITER, cleanup); + /* Add NOP to ensure correct line tracing of multiline for statements. + * It will be removed later if redundant. + */ + ADDOP(c, LOC(s->v.For.target), NOP); + USE_LABEL(c, body); VISIT(c, expr, s->v.For.target); VISIT_SEQ(c, stmt, s->v.For.body); From ce3879bd45e068f8e2a5a214acd234ca44cad53b Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Wed, 12 Jun 2024 20:24:43 +0800 Subject: [PATCH 455/903] Fix typos in documentation (#120338) --- InternalDocs/compiler.md | 2 +- Lib/idlelib/HISTORY.txt | 2 +- Lib/idlelib/News3.txt | 6 +++--- Lib/idlelib/TODO.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 0abc10da6e05c6..17fe0df6e1db10 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -625,7 +625,7 @@ Objects * [Objects/locations.md](https://github.com/python/cpython/blob/main/Objects/locations.md): Describes the location table * [Objects/frame_layout.md](https://github.com/python/cpython/blob/main/Objects/frame_layout.md): Describes the frame stack -* [Objects/object_layout.md](https://github.com/python/cpython/blob/main/Objects/object_layout.md): Descibes object layout for 3.11 and later +* [Objects/object_layout.md](https://github.com/python/cpython/blob/main/Objects/object_layout.md): Describes object layout for 3.11 and later * [Exception Handling](exception_handling.md): Describes the exception table diff --git a/Lib/idlelib/HISTORY.txt b/Lib/idlelib/HISTORY.txt index 731fabd185fbbf..a601b25b5f838f 100644 --- a/Lib/idlelib/HISTORY.txt +++ b/Lib/idlelib/HISTORY.txt @@ -277,7 +277,7 @@ Command to format a paragraph. Debug menu: JIT (Just-In-Time) stack viewer toggle -- if set, the stack viewer -automaticall pops up when you get a traceback. +automatically pops up when you get a traceback. Windows menu: diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index fb07d7b3b3fad8..b1b652dc562c8e 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -568,14 +568,14 @@ bpo-33679: Enable theme-specific color configuration for Code Context. color setting, default or custom, on the extensions tab, that applied to all themes.) For built-in themes, the foreground is the same as normal text and the background is a contrasting gray. Context colors for -custom themes are set on the Hightlights tab along with other colors. +custom themes are set on the Highlights tab along with other colors. When one starts IDLE from a console and loads a custom theme without definitions for 'context', one will see a warning message on the console. bpo-33642: Display up to maxlines non-blank lines for Code Context. If there is no current context, show a single blank line. (Previously, -the Code Contex had numlines lines, usually with some blank.) The use +the Code Context had numlines lines, usually with some blank.) The use of a new option, 'maxlines' (default 15), avoids possible interference with user settings of the old option, 'numlines' (default 3). @@ -729,7 +729,7 @@ not affect their keyset-specific customization after 3.6.3. and vice versa. Initial patch by Charles Wohlganger, revised by Terry Jan Reedy. -bpo-31051: Rearrange condigdialog General tab. +bpo-31051: Rearrange configdialog General tab. Sort non-Help options into Window (Shell+Editor) and Editor (only). Leave room for the addition of new options. Patch by Terry Jan Reedy. diff --git a/Lib/idlelib/TODO.txt b/Lib/idlelib/TODO.txt index e2f1ac0f274001..41b86b0c6d5bbd 100644 --- a/Lib/idlelib/TODO.txt +++ b/Lib/idlelib/TODO.txt @@ -179,7 +179,7 @@ it -- i.e. you can only edit the current command, and the cursor can't escape from the command area. (Albert Brandl) - Set X11 class to "idle/Idle", set icon and title to something -beginning with "idle" -- for window manangers. (Randall Hopper) +beginning with "idle" -- for window managers. (Randall Hopper) - Config files editable through a preferences dialog. (me) DONE From e16aed63f64b18a26859eff3de976ded373e66b8 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Wed, 12 Jun 2024 20:41:07 +0800 Subject: [PATCH 456/903] gh-117657: Make Py_TYPE and Py_SET_TYPE thread safe (GH-120165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Nadeshiko Manju --- Include/internal/pycore_interp.h | 5 ++++- Include/object.h | 8 +++++++ Lib/test/test_free_threading/test_type.py | 26 ++++++++++++++++++++++ Objects/typeobject.c | 8 ++++++- Tools/tsan/suppressions_free_threading.txt | 2 -- 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 86dada5061e7b5..6b5f50b88f7b85 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -401,7 +401,10 @@ PyAPI_FUNC(PyStatus) _PyInterpreterState_New( #define RARE_EVENT_INTERP_INC(interp, name) \ do { \ /* saturating add */ \ - if (interp->rare_events.name < UINT8_MAX) interp->rare_events.name++; \ + int val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \ + if (val < UINT8_MAX) { \ + FT_ATOMIC_STORE_UINT8(interp->rare_events.name, val + 1); \ + } \ RARE_EVENT_STAT_INC(name); \ } while (0); \ diff --git a/Include/object.h b/Include/object.h index c8c63b9b2b1450..4a39ada8c7daa4 100644 --- a/Include/object.h +++ b/Include/object.h @@ -246,7 +246,11 @@ _Py_IsOwnedByCurrentThread(PyObject *ob) // bpo-39573: The Py_SET_TYPE() function must be used to set an object type. static inline PyTypeObject* Py_TYPE(PyObject *ob) { +#ifdef Py_GIL_DISABLED + return (PyTypeObject *)_Py_atomic_load_ptr_relaxed(&ob->ob_type); +#else return ob->ob_type; +#endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_TYPE(ob) Py_TYPE(_PyObject_CAST(ob)) @@ -274,7 +278,11 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ptr(&ob->ob_type, type); +#else ob->ob_type = type; +#endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 786336fa0cddce..1e84b2db2d4882 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -96,6 +96,32 @@ def reader_func(): self.run_one(writer_func, reader_func) + def test___class___modification(self): + class Foo: + pass + + class Bar: + pass + + thing = Foo() + def work(): + foo = thing + for _ in range(10000): + foo.__class__ = Bar + type(foo) + foo.__class__ = Foo + type(foo) + + + threads = [] + for i in range(NTHREADS): + thread = threading.Thread(target=work) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + def run_one(self, writer_func, reader_func): writer = Thread(target=writer_func) readers = [] diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 070e3d2f7bf2b4..8ecab555454cdc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6633,9 +6633,15 @@ object_set_class(PyObject *self, PyObject *value, void *closure) if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_INCREF(newto); } + Py_BEGIN_CRITICAL_SECTION(self); + // The real Py_TYPE(self) (`oldto`) may have changed from + // underneath us in another thread, so we re-fetch it here. + oldto = Py_TYPE(self); Py_SET_TYPE(self, newto); - if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) + Py_END_CRITICAL_SECTION(); + if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_DECREF(oldto); + } RARE_EVENT_INC(set_class); return 0; diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index cb48a30751ac7b..b10b297f50da81 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -37,7 +37,6 @@ race_top:set_contains_key # https://gist.github.com/colesbury/d13d033f413b4ad07929d044bed86c35 race_top:set_discard_entry race_top:set_inheritable -race_top:Py_SET_TYPE race_top:_PyDict_CheckConsistency race_top:_Py_dict_lookup_threadsafe race_top:_multiprocessing_SemLock_acquire_impl @@ -58,7 +57,6 @@ race_top:_PyFrame_Initialize race_top:PyInterpreterState_ThreadHead race_top:_PyObject_TryGetInstanceAttribute race_top:PyThreadState_Next -race_top:Py_TYPE race_top:PyUnstable_InterpreterFrame_GetLine race_top:sock_close race_top:tstate_delete_common From 32d3e05fe67d43f7285e582a87e65374cf7c2972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 12 Jun 2024 15:23:45 +0200 Subject: [PATCH 457/903] gh-120029: remove unused macros in ``symtable.c`` (#120222) Co-authored-by: Carl Meyer Co-authored-by: Jelle Zijlstra --- Include/internal/pycore_symtable.h | 4 ---- Modules/symtablemodule.c | 1 - Python/symtable.c | 1 - 3 files changed, 6 deletions(-) diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 1be48edc80c830..519505c0edf52a 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -143,7 +143,6 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define DEF_PARAM (2<<1) /* formal parameter */ #define DEF_NONLOCAL (2<<2) /* nonlocal stmt */ #define USE (2<<3) /* name is used */ -#define DEF_FREE (2<<4) /* name used but not defined in nested block */ #define DEF_FREE_CLASS (2<<5) /* free variable from class's method */ #define DEF_IMPORT (2<<6) /* assignment occurred via import */ #define DEF_ANNOT (2<<7) /* this name is annotated */ @@ -166,9 +165,6 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define FREE 4 #define CELL 5 -#define GENERATOR 1 -#define GENERATOR_EXPRESSION 2 - // Used by symtablemodule.c extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index b39b59bf7b06bf..618465536e7851 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -76,7 +76,6 @@ symtable_init_constants(PyObject *m) if (PyModule_AddIntMacro(m, DEF_LOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_PARAM) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_TYPE_PARAM) < 0) return -1; - if (PyModule_AddIntMacro(m, DEF_FREE) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_FREE_CLASS) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_IMPORT) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_BOUND) < 0) return -1; diff --git a/Python/symtable.c b/Python/symtable.c index 627184da9ef4ed..0490014166e65c 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -327,7 +327,6 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) if (flags & DEF_PARAM) printf(" DEF_PARAM"); if (flags & DEF_NONLOCAL) printf(" DEF_NONLOCAL"); if (flags & USE) printf(" USE"); - if (flags & DEF_FREE) printf(" DEF_FREE"); if (flags & DEF_FREE_CLASS) printf(" DEF_FREE_CLASS"); if (flags & DEF_IMPORT) printf(" DEF_IMPORT"); if (flags & DEF_ANNOT) printf(" DEF_ANNOT"); From 4b1e85bafc5bcb8cb70bb17164e07aebf7ad7e8e Mon Sep 17 00:00:00 2001 From: ixgbe00 Date: Wed, 12 Jun 2024 21:24:46 +0800 Subject: [PATCH 458/903] =?UTF-8?q?gh-120400=20=EF=BC=9ASupport=20Linux=20?= =?UTF-8?q?perf=20profile=20to=20see=20Python=20calls=20on=20RISC-V=20arch?= =?UTF-8?q?itecture=20(#120089)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Pablo Galindo Salgado --- .../next/Core and Builtins/2024-06-05-06-26-04.gh-issue- | 1 + .../2024-06-12-12-29-45.gh-issue-120400.lZYHVS.rst | 1 + Python/asm_trampoline.S | 8 ++++++++ configure | 2 ++ configure.ac | 1 + 5 files changed, 13 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-05-06-26-04.gh-issue- create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-12-12-29-45.gh-issue-120400.lZYHVS.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-05-06-26-04.gh-issue- b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-06-26-04.gh-issue- new file mode 100644 index 00000000000000..29f06d43c3598c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-05-06-26-04.gh-issue- @@ -0,0 +1 @@ +Support Linux perf profiler to see Python calls on RISC-V architecture diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-12-12-29-45.gh-issue-120400.lZYHVS.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-12-29-45.gh-issue-120400.lZYHVS.rst new file mode 100644 index 00000000000000..8c86d4750e39a8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-12-29-45.gh-issue-120400.lZYHVS.rst @@ -0,0 +1 @@ +Support Linux perf profiler to see Python calls on RISC-V architecture. diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index 460707717df003..0a3265dfeee204 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -22,6 +22,14 @@ _Py_trampoline_func_start: blr x3 ldp x29, x30, [sp], 16 ret +#endif +#ifdef __riscv + addi sp,sp,-16 + sd ra,8(sp) + jalr a3 + ld ra,8(sp) + addi sp,sp,16 + jr ra #endif .globl _Py_trampoline_func_end _Py_trampoline_func_end: diff --git a/configure b/configure index 8e605d31bb5eca..4174633b51c30a 100755 --- a/configure +++ b/configure @@ -13133,6 +13133,8 @@ case $PLATFORM_TRIPLET in #( perf_trampoline=yes ;; #( aarch64-linux-gnu) : perf_trampoline=yes ;; #( + riscv64-linux-gnu) : + perf_trampoline=yes ;; #( *) : perf_trampoline=no ;; diff --git a/configure.ac b/configure.ac index 41023ab92bad81..d34ade389cf40c 100644 --- a/configure.ac +++ b/configure.ac @@ -3641,6 +3641,7 @@ AC_MSG_CHECKING([perf trampoline]) AS_CASE([$PLATFORM_TRIPLET], [x86_64-linux-gnu], [perf_trampoline=yes], [aarch64-linux-gnu], [perf_trampoline=yes], + [riscv64-linux-gnu], [perf_trampoline=yes], [perf_trampoline=no] ) AC_MSG_RESULT([$perf_trampoline]) From 42b25dd61ff3593795c4cc2ffe876ab766098b24 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 12 Jun 2024 15:27:07 +0200 Subject: [PATCH 459/903] gh-120155: Add assertion to sre.c match_getindex() (#120402) Add an assertion to help static analyzers to detect that i*2 cannot overflow. --- Modules/_sre/sre.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index e33034086481c2..0c656b47991c2f 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -2217,6 +2217,8 @@ match_getindex(MatchObject* self, PyObject* index) return -1; } + // Check that i*2 cannot overflow to make static analyzers happy + assert(i <= SRE_MAXGROUPS); return i; } From 92c9c6ae147e1e658bbc8d454f8c7b2c4dea31d1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 12 Jun 2024 17:23:03 +0300 Subject: [PATCH 460/903] gh-120345: Fix incorrect use of the :class: role with the "()" suffix (GH-120347) * Remove "()" when refer to a class as a type. * Use :func: when refer to a callable. * Fix reference to the datetime.astimezone() method. --- Doc/howto/descriptor.rst | 2 +- Doc/library/collections.rst | 2 +- Doc/library/datetime.rst | 2 +- Doc/library/fileinput.rst | 2 +- Doc/tutorial/stdlib2.rst | 4 ++-- Doc/whatsnew/2.5.rst | 4 ++-- Doc/whatsnew/3.12.rst | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 51f9f4a6556e57..b29488be39a0a3 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -787,7 +787,7 @@ Invocation from super --------------------- The logic for super's dotted lookup is in the :meth:`__getattribute__` method for -object returned by :class:`super()`. +object returned by :func:`super`. A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` for the base class ``B`` immediately following ``A`` and then returns diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 2a269712f1814d..ce89101d6b667c 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -99,7 +99,7 @@ The class can be used to simulate nested scopes and is useful in templating. :func:`super` function. A reference to ``d.parents`` is equivalent to: ``ChainMap(*d.maps[1:])``. - Note, the iteration order of a :class:`ChainMap()` is determined by + Note, the iteration order of a :class:`ChainMap` is determined by scanning the mappings last to first:: >>> baseline = {'music': 'bach', 'art': 'rembrandt'} diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 0723d0fe2fceb0..b6d8e6e6df07fa 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2153,7 +2153,7 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: .. method:: tzinfo.fromutc(dt) - This is called from the default :class:`datetime.astimezone()` + This is called from the default :meth:`datetime.astimezone` implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s date and time data are to be viewed as expressing a UTC time. The purpose of :meth:`fromutc` is to adjust the date and time data, returning an diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 94a4139f64c2e4..8f32b11e565365 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -47,7 +47,7 @@ Lines are returned with any newlines intact, which means that the last line in a file may not have one. You can control how files are opened by providing an opening hook via the -*openhook* parameter to :func:`fileinput.input` or :class:`FileInput()`. The +*openhook* parameter to :func:`fileinput.input` or :func:`FileInput`. The hook must be a function that takes two arguments, *filename* and *mode*, and returns an accordingly opened file-like object. If *encoding* and/or *errors* are specified, they will be passed to the hook as additional keyword arguments. diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 4bc810ce36c71b..719f772e687008 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -293,7 +293,7 @@ Many data structure needs can be met with the built-in list type. However, sometimes there is a need for alternative implementations with different performance trade-offs. -The :mod:`array` module provides an :class:`~array.array()` object that is like +The :mod:`array` module provides an :class:`~array.array` object that is like a list that stores only homogeneous data and stores it more compactly. The following example shows an array of numbers stored as two byte unsigned binary numbers (typecode ``"H"``) rather than the usual 16 bytes per entry for regular @@ -306,7 +306,7 @@ lists of Python int objects:: >>> a[1:3] array('H', [10, 700]) -The :mod:`collections` module provides a :class:`~collections.deque()` object +The :mod:`collections` module provides a :class:`~collections.deque` object that is like a list with faster appends and pops from the left side but slower lookups in the middle. These objects are well suited for implementing queues and breadth first tree searches:: diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 2ae26e7a106a0b..3430ac8668e280 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1724,7 +1724,7 @@ attribute of the function object to change this:: :mod:`ctypes` also provides a wrapper for Python's C API as the ``ctypes.pythonapi`` object. This object does *not* release the global interpreter lock before calling a function, because the lock must be held when -calling into the interpreter's code. There's a :class:`py_object()` type +calling into the interpreter's code. There's a :class:`~ctypes.py_object` type constructor that will create a :c:expr:`PyObject *` pointer. A simple usage:: import ctypes @@ -1734,7 +1734,7 @@ constructor that will create a :c:expr:`PyObject *` pointer. A simple usage:: ctypes.py_object("abc"), ctypes.py_object(1)) # d is now {'abc', 1}. -Don't forget to use :class:`py_object()`; if it's omitted you end up with a +Don't forget to use :func:`~ctypes.py_object`; if it's omitted you end up with a segmentation fault. :mod:`ctypes` has been around for a while, but people still write and diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 28b28e9ce50e11..93d18ffc76d07c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -739,7 +739,7 @@ inspect itertools --------- -* Add :class:`itertools.batched()` for collecting into even-sized +* Add :func:`itertools.batched` for collecting into even-sized tuples where the last batch may be shorter than the rest. (Contributed by Raymond Hettinger in :gh:`98363`.) From fabcf6bc8f89f008319442dea614d5cbeb959544 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 12 Jun 2024 17:50:58 +0300 Subject: [PATCH 461/903] gh-120388: Improve deprecation warning message, when test returns non-None (#120401) Co-authored-by: Alex Waygood Co-authored-by: Serhiy Storchaka --- Lib/test/test_unittest/test_async_case.py | 3 +++ Lib/test/test_unittest/test_case.py | 19 +++++++++++++++++++ Lib/unittest/async_case.py | 10 +++++++--- Lib/unittest/case.py | 15 ++++++++++++--- ...-06-12-15-07-58.gh-issue-120388.VuTQMT.rst | 3 +++ 5 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index ba1ab838cd4a22..00ef55bdf9bc83 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -312,18 +312,21 @@ async def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'async_generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) def test_cleanups_interleave_order(self): events = [] diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index ed5eb5609a5dd1..17420909402107 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -325,18 +325,37 @@ def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) + + def test_deprecation_of_return_val_from_test_async_method(self): + class Foo(unittest.TestCase): + async def test1(self): + return 1 + + with self.assertWarns(DeprecationWarning) as w: + Foo('test1').run() + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) + self.assertIn('test1', str(w.warning)) + self.assertEqual(w.filename, __file__) + self.assertIn("returned 'coroutine'", str(w.warning)) + self.assertIn( + 'Maybe you forgot to use IsolatedAsyncioTestCase as the base class?', + str(w.warning), + ) def _check_call_order__subtests(self, result, events, expected_events): class Foo(Test.LoggingTestCase): diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index 63ff6a5d1f8b61..bd06eb3207697a 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -90,9 +90,13 @@ def _callSetUp(self): self._callAsync(self.asyncSetUp) def _callTestMethod(self, method): - if self._callMaybeAsync(method) is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=4) + result = self._callMaybeAsync(method) + if result is not None: + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})', + ) + warnings.warn(msg, DeprecationWarning, stacklevel=4) def _callTearDown(self): self._callAsync(self.asyncTearDown) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 36daa61fa31adb..55c79d353539ca 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -603,9 +603,18 @@ def _callSetUp(self): self.setUp() def _callTestMethod(self, method): - if method() is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=3) + result = method() + if result is not None: + import inspect + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})' + ) + if inspect.iscoroutine(result): + msg += ( + '. Maybe you forgot to use IsolatedAsyncioTestCase as the base class?' + ) + warnings.warn(msg, DeprecationWarning, stacklevel=3) def _callTearDown(self): self.tearDown() diff --git a/Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst b/Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst new file mode 100644 index 00000000000000..d13df7d88b776c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-12-15-07-58.gh-issue-120388.VuTQMT.rst @@ -0,0 +1,3 @@ +Improve a warning message when a test method in :mod:`unittest` returns +something other than ``None``. Now we show the returned object type and +optional asyncio-related tip. From 127c1d2771749853e287632c086b6054212bf12a Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 13 Jun 2024 01:46:39 +0900 Subject: [PATCH 462/903] gh-71587: Drop local reference cache to `_strptime` module in `_datetime` (gh-120224) The _strptime module object was cached in a static local variable (in the datetime.strptime() implementation). That's a problem when it crosses isolation boundaries, such as reinitializing the runtme or between interpreters. This change fixes the problem by dropping the static variable, instead always relying on the normal sys.modules cache (via PyImport_Import()). --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_runtime_init_generated.h | 1 + Include/internal/pycore_unicodeobject_generated.h | 3 +++ Lib/test/test_embed.py | 9 +++++++++ .../2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst | 2 ++ Modules/_datetimemodule.c | 14 +++++++------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 8 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 30851dc2dbec44..bc94930b85f098 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -777,6 +777,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_showwarnmsg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_shutdown)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_slotnames)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_swappedbytes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_type_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 009802c441685c..998be2ec490dd9 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -266,6 +266,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_showwarnmsg) STRUCT_FOR_ID(_shutdown) STRUCT_FOR_ID(_slotnames) + STRUCT_FOR_ID(_strptime) STRUCT_FOR_ID(_strptime_datetime) STRUCT_FOR_ID(_swappedbytes_) STRUCT_FOR_ID(_type_) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index ff5b6ee8e0f006..bd79a7dff42f89 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -775,6 +775,7 @@ extern "C" { INIT_ID(_showwarnmsg), \ INIT_ID(_shutdown), \ INIT_ID(_slotnames), \ + INIT_ID(_strptime), \ INIT_ID(_strptime_datetime), \ INIT_ID(_swappedbytes_), \ INIT_ID(_type_), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 69d93a9610a2e5..7284aeb592d7ec 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -636,6 +636,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_slotnames); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(_strptime); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_strptime_datetime); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d94c63a13b8ea4..634513ec7a5812 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -404,6 +404,15 @@ def test_ucnhash_capi_reset(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '9\n' * INIT_LOOPS) + def test_datetime_reset_strptime(self): + code = ( + "import datetime;" + "d = datetime.datetime.strptime('2000-01-01', '%Y-%m-%d');" + "print(d.strftime('%Y%m%d'))" + ) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '20000101\n' * INIT_LOOPS) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst b/Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst new file mode 100644 index 00000000000000..50a662977993f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-07-11-23-31.gh-issue-71587.IjFajE.rst @@ -0,0 +1,2 @@ +Fix crash in C version of :meth:`datetime.datetime.strptime` when called again +on the restarted interpreter. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index bea6e9411a75ed..cb4622893375d7 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5514,19 +5514,19 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) static PyObject * datetime_strptime(PyObject *cls, PyObject *args) { - static PyObject *module = NULL; - PyObject *string, *format; + PyObject *string, *format, *result; if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) return NULL; + PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { - module = PyImport_ImportModule("_strptime"); - if (module == NULL) - return NULL; + return NULL; } - return PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime), - cls, string, format, NULL); + result = PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime), + cls, string, format, NULL); + Py_DECREF(module); + return result; } /* Return new datetime from date/datetime and time arguments. */ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 4586a59f6ac2ef..cb9750a69a632b 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -393,7 +393,6 @@ Modules/xxmodule.c - ErrorObject - ## initialized once Modules/_cursesmodule.c - ModDict - -Modules/_datetimemodule.c datetime_strptime module - ## state Modules/_datetimemodule.c - _datetime_global_state - From 4b5d3e0e721a952f4ac9d17bee331e6dfe543dcd Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 12 Jun 2024 20:52:55 +0200 Subject: [PATCH 463/903] gh-120343: Fix column offsets of multiline tokens in tokenize (#120391) --- Lib/test/test_tokenize.py | 14 ++++++++++++++ Python/Python-tokenize.c | 14 ++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 36dba71766cc20..51aeb35f01065a 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1210,6 +1210,20 @@ def test_multiline_non_ascii_fstring(self): FSTRING_END "\'\'\'" (2, 68) (2, 71) """) + def test_multiline_non_ascii_fstring_with_expr(self): + self.check_tokenize("""\ +f''' + 🔗 This is a test {test_arg1}🔗 +🔗'''""", """\ + FSTRING_START "f\'\'\'" (1, 0) (1, 4) + FSTRING_MIDDLE '\\n 🔗 This is a test ' (1, 4) (2, 21) + OP '{' (2, 21) (2, 22) + NAME 'test_arg1' (2, 22) (2, 31) + OP '}' (2, 31) (2, 32) + FSTRING_MIDDLE '🔗\\n🔗' (2, 32) (3, 1) + FSTRING_END "\'\'\'" (3, 1) (3, 4) + """) + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 2591dae35736ba..55c821754c2031 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -215,6 +215,7 @@ tokenizeriter_next(tokenizeriterobject *it) const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start; PyObject* line = NULL; + int line_changed = 1; if (it->tok->tok_extra_tokens && is_trailing_token) { line = PyUnicode_FromString(""); } else { @@ -229,12 +230,11 @@ tokenizeriter_next(tokenizeriterobject *it) Py_XDECREF(it->last_line); line = PyUnicode_DecodeUTF8(line_start, size, "replace"); it->last_line = line; - if (it->tok->lineno != it->last_end_lineno) { - it->byte_col_offset_diff = 0; - } + it->byte_col_offset_diff = 0; } else { // Line hasn't changed so we reuse the cached one. line = it->last_line; + line_changed = 0; } } if (line == NULL) { @@ -252,7 +252,13 @@ tokenizeriter_next(tokenizeriterobject *it) Py_ssize_t byte_offset = -1; if (token.start != NULL && token.start >= line_start) { byte_offset = token.start - line_start; - col_offset = byte_offset - it->byte_col_offset_diff; + if (line_changed) { + col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); + it->byte_col_offset_diff = byte_offset - col_offset; + } + else { + col_offset = byte_offset - it->byte_col_offset_diff; + } } if (token.end != NULL && token.end >= it->tok->line_start) { Py_ssize_t end_byte_offset = token.end - it->tok->line_start; From 4c6d4f5cb33e48519922d635894eef356faddba2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 12 Jun 2024 20:56:42 +0200 Subject: [PATCH 464/903] gh-120417: Remove unused imports in the stdlib (#120420) --- Lib/_pyrepl/historical_reader.py | 2 +- Lib/_pyrepl/pager.py | 2 +- Lib/_pyrepl/unix_console.py | 1 - Lib/_pyrepl/windows_console.py | 3 --- Lib/dataclasses.py | 3 +-- Lib/dbm/sqlite3.py | 1 - Lib/idlelib/grep.py | 2 +- Lib/importlib/abc.py | 1 - Lib/ntpath.py | 1 - Lib/pydoc.py | 2 +- Lib/stat.py | 1 - 11 files changed, 5 insertions(+), 14 deletions(-) diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index 121de33da5052f..dd90912d1d67f8 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -27,7 +27,7 @@ if False: - from .types import Callback, SimpleContextManager, KeySpec, CommandName + from .types import SimpleContextManager, KeySpec, CommandName isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index 1ac733ed3573a4..66dcd99111adfc 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -8,7 +8,7 @@ # types if False: - from typing import Protocol, Any + from typing import Protocol class Pager(Protocol): def __call__(self, text: str, title: str = "") -> None: ... diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index af9290819c2c78..c4dedd97d1e13d 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -27,7 +27,6 @@ import select import signal import struct -import sys import termios import time from fcntl import ioctl diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index f691ca3fbb07b8..9e97b1524e29a0 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -20,15 +20,12 @@ from __future__ import annotations import io -from multiprocessing import Value import os import sys import time import msvcrt -from abc import ABC, abstractmethod from collections import deque -from dataclasses import dataclass, field import ctypes from ctypes.wintypes import ( _COORD, diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index aeafbfbbe6e9c4..74011b7e28b9f3 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -7,7 +7,6 @@ import itertools import abc from reprlib import recursive_repr -from types import FunctionType, GenericAlias __all__ = ['dataclass', @@ -333,7 +332,7 @@ def __set_name__(self, owner, name): # it. func(self.default, owner, name) - __class_getitem__ = classmethod(GenericAlias) + __class_getitem__ = classmethod(types.GenericAlias) class _DataclassParams: diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 74c9d9b7e2f1d8..7e0ae2a29e3a64 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -1,6 +1,5 @@ import os import sqlite3 -import sys from pathlib import Path from contextlib import suppress, closing from collections.abc import MutableMapping diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py index ef14349960bfa2..42048ff2395fe1 100644 --- a/Lib/idlelib/grep.py +++ b/Lib/idlelib/grep.py @@ -190,7 +190,7 @@ def grep_it(self, prog, path): def _grep_dialog(parent): # htest # - from tkinter import Toplevel, Text, SEL, END + from tkinter import Toplevel, Text, SEL from tkinter.ttk import Frame, Button from idlelib.pyshell import PyShellFileList diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index b6b2c791a3b03f..eea6b38af6fa13 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -13,7 +13,6 @@ _frozen_importlib_external = _bootstrap_external from ._abc import Loader import abc -import warnings __all__ = [ diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 83e2d3b865757c..1b1873f08b608b 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -19,7 +19,6 @@ import os import sys -import stat import genericpath from genericpath import * diff --git a/Lib/pydoc.py b/Lib/pydoc.py index d7579c1cc3dcd1..278e4846ebb71f 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -75,7 +75,7 @@ class or function within a module or module in a package. If the from reprlib import Repr from traceback import format_exception_only -from _pyrepl.pager import (get_pager, plain, escape_less, pipe_pager, +from _pyrepl.pager import (get_pager, plain, pipe_pager, plain_pager, tempfile_pager, tty_pager) diff --git a/Lib/stat.py b/Lib/stat.py index 9167ab185944fb..1b4ed1ebc940ef 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -2,7 +2,6 @@ Suggested usage: from stat import * """ -import sys # Indices for stat struct members in the tuple returned by os.stat() From 3453362183f083e37ea866a7ae1b34147ffaf81d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 12 Jun 2024 20:09:25 +0100 Subject: [PATCH 465/903] gh-118908: Protect the REPL subprocess with a timeout in tests (#120408) --- Lib/test/test_pyrepl/test_pyrepl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 3167b8473bfe20..41ba5959a1ec34 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -8,6 +8,7 @@ from unittest import TestCase, skipUnless from unittest.mock import patch from test.support import force_not_colorized +from test.support import SHORT_TIMEOUT from .support import ( FakeConsole, @@ -885,5 +886,9 @@ def run_repl(self, repl_input: str | list[str], env: dict | None = None) -> tupl os.close(master_fd) os.close(slave_fd) - exit_code = process.wait() + try: + exit_code = process.wait(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + exit_code = process.returncode return "\n".join(output), exit_code From 030b452e34bbb0096acacb70a31915b9590c8186 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Wed, 12 Jun 2024 12:19:36 -0700 Subject: [PATCH 466/903] gh-120418: Don't assume wheeldata is deleted if `WHEEL_PKG_DIR` is set (#120419) Remove wheeldata from both sides of the `assertEqual`, so that we're *actually* ignoring it from the test set. This test is only making assertions about the source tree, no code is being executed that would do anything different based on the value of `WHEEL_PKG_DIR`. --- Lib/test/test_tools/test_makefile.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index 48a7c1a773bb83..df95e6d0068516 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -41,7 +41,7 @@ def test_makefile_test_folders(self): idle_test = 'idlelib/idle_test' self.assertIn(idle_test, test_dirs) - used = [idle_test] + used = set([idle_test]) for dirpath, dirs, files in os.walk(support.TEST_HOME_DIR): dirname = os.path.basename(dirpath) # Skip temporary dirs: @@ -65,13 +65,14 @@ def test_makefile_test_folders(self): "of test directories to install" ) ) - used.append(relpath) + used.add(relpath) # Don't check the wheel dir when Python is built --with-wheel-pkg-dir if sysconfig.get_config_var('WHEEL_PKG_DIR'): test_dirs.remove('test/wheeldata') + used.discard('test/wheeldata') # Check that there are no extra entries: unique_test_dirs = set(test_dirs) - self.assertSetEqual(unique_test_dirs, set(used)) + self.assertSetEqual(unique_test_dirs, used) self.assertEqual(len(test_dirs), len(unique_test_dirs)) From eebae2c460dabdc70dc0d9b6e189368eb1abb716 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Thu, 13 Jun 2024 17:29:19 +0800 Subject: [PATCH 467/903] gh-117657: Make PyType_HasFeature atomic (GH-120210) Make PyType_HasFeature atomic --- Include/internal/pycore_object.h | 2 +- Include/internal/pycore_pyatomic_ft_wrappers.h | 3 +++ Tools/tsan/suppressions_free_threading.txt | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 6f133014ce06e2..d1e2773a2473b0 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -262,7 +262,7 @@ extern int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void*); // Fast inlined version of PyType_HasFeature() static inline int _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { - return ((type->tp_flags & feature) != 0); + return ((FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags) & feature) != 0); } extern void _PyType_InitCache(PyInterpreterState *interp); diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index bc6aba56cf9fc7..a1bb383bcd22e9 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -45,6 +45,8 @@ extern "C" { _Py_atomic_load_uint16_relaxed(&value) #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ + _Py_atomic_load_ulong_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ _Py_atomic_store_ptr_relaxed(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ @@ -75,6 +77,7 @@ extern "C" { #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index b10b297f50da81..05ceaf438b6353 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -26,7 +26,6 @@ race:free_threadstate race_top:_add_to_weak_set race_top:_in_weak_set race_top:_PyEval_EvalFrameDefault -race_top:_PyType_HasFeature race_top:assign_version_tag race_top:insertdict race_top:lookup_tp_dict From b1b61dc4cee43920ef2b08d5ac94ddf08119c507 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Thu, 13 Jun 2024 17:31:21 +0800 Subject: [PATCH 468/903] gh-117657: Fix some simple races in instrumentation.c (GH-120118) * stop the world when setting local events --- Python/instrumentation.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index a5211ee5428cf8..ae790a1441b933 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1977,7 +1977,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent } int res; - LOCK_CODE(code); + _PyEval_StopTheWorld(interp); if (allocate_instrumentation_data(code)) { res = -1; goto done; @@ -1994,7 +1994,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent res = force_instrument_lock_held(code, interp); done: - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); return res; } From 87cedaa5c857dc615b9f618b020414187fc1c966 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Thu, 13 Jun 2024 17:37:21 +0800 Subject: [PATCH 469/903] Fix typos in documentation (GH-120440) --- .../2024-06-10-10-42-48.gh-issue-120298.napREA.rst | 2 +- .../2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst index 531d39517ac423..2872006ee34b8b 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst @@ -1,2 +1,2 @@ Fix use-after free in ``list_richcompare_impl`` which can be invoked via -some specificly tailored evil input. +some specifically tailored evil input. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst index eb2d0f9a705caa..757a21625cfb83 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-11-12-47-54.gh-issue-120346.hhn_6X.rst @@ -1,2 +1,2 @@ -Respect :envvar:`PYTHON_BASIC_REPL` when running in interative inspect mode +Respect :envvar:`PYTHON_BASIC_REPL` when running in interactive inspect mode (``python -i``). Patch by Pablo Galindo From ca5108a46d5da3978d4bd29717ea3fbdee772e66 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 13 Jun 2024 14:38:31 +0300 Subject: [PATCH 470/903] gh-119146: Update ``regexp`` in `build.yml` to not trigger the jobs on `*.md` and `*.ini` files. (#120435) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb325ac2f9ee1b..750aa1ed87bca1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,7 +54,7 @@ jobs: # into the PR branch anyway. # # https://github.com/python/core-workflow/issues/373 - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true fi # Check if we should run hypothesis tests From 6ae254aaa0a5a3985a52d1ab387a2b68c001bd96 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Jun 2024 16:14:50 +0200 Subject: [PATCH 471/903] gh-120417: Add #noqa to used imports in the stdlib (#120421) Tools such as ruff can ignore "imported but unused" warnings if a line ends with "# noqa: F401". It avoids the temptation to remove an import which is used effectively. --- Lib/_pyio.py | 2 +- Lib/code.py | 2 +- Lib/codecs.py | 2 +- Lib/collections/__init__.py | 3 ++- Lib/concurrent/futures/process.py | 2 +- Lib/curses/__init__.py | 2 +- Lib/datetime.py | 4 ++-- Lib/decimal.py | 8 ++++---- Lib/hashlib.py | 2 +- Lib/lzma.py | 2 +- Lib/multiprocessing/context.py | 2 +- Lib/multiprocessing/util.py | 2 +- Lib/opcode.py | 4 ++-- Lib/operator.py | 2 +- Lib/platform.py | 2 +- Lib/pstats.py | 2 +- Lib/pydoc.py | 5 ++++- Lib/re/_constants.py | 2 +- Lib/site.py | 6 +++--- Lib/sqlite3/__main__.py | 2 +- Lib/struct.py | 4 ++-- Lib/symtable.py | 6 +++--- Lib/unittest/__init__.py | 4 ++-- Lib/urllib/request.py | 2 +- Lib/xml/dom/__init__.py | 2 +- 25 files changed, 40 insertions(+), 36 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index a3fede699218a1..7d298e1674b49a 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -16,7 +16,7 @@ _setmode = None import io -from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) # noqa: F401 valid_seek_flags = {0, 1, 2} # Hardwired values if hasattr(os, 'SEEK_HOLE') : diff --git a/Lib/code.py b/Lib/code.py index b93902ccf545b3..a55fced0704b1d 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -355,7 +355,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa console.raw_input = readfunc else: try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg) diff --git a/Lib/codecs.py b/Lib/codecs.py index 9b35b6127dd01c..a887e5d4c94a38 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -1129,4 +1129,4 @@ def make_encoding_map(decoding_map): # package _false = 0 if _false: - import encodings + import encodings # noqa: F401 diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index a17100e6c02a0e..b47e728484c8ac 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -46,7 +46,8 @@ _collections_abc.MutableSequence.register(deque) try: - from _collections import _deque_iterator + # Expose _deque_iterator to support pickling deque iterators + from _collections import _deque_iterator # noqa: F401 except ImportError: pass diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index bb4892ebdfedf5..7092b4757b5429 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -589,7 +589,7 @@ def _check_system_limits(): raise NotImplementedError(_system_limited) _system_limits_checked = True try: - import multiprocessing.synchronize + import multiprocessing.synchronize # noqa: F401 except ImportError: _system_limited = ( "This Python build lacks multiprocessing.synchronize, usually due " diff --git a/Lib/curses/__init__.py b/Lib/curses/__init__.py index 69270bfcd2b205..6165fe6c9875c0 100644 --- a/Lib/curses/__init__.py +++ b/Lib/curses/__init__.py @@ -53,7 +53,7 @@ def start_color(): try: has_key except NameError: - from .has_key import has_key + from .has_key import has_key # noqa: F401 # Wrapper for the entire curses-based application. Runs a function which # should be the rest of your curses-based application. If the application diff --git a/Lib/datetime.py b/Lib/datetime.py index a33d2d724cb33d..b4f7bd045c7b68 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1,9 +1,9 @@ try: from _datetime import * - from _datetime import __doc__ + from _datetime import __doc__ # noqa: F401 except ImportError: from _pydatetime import * - from _pydatetime import __doc__ + from _pydatetime import __doc__ # noqa: F401 __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") diff --git a/Lib/decimal.py b/Lib/decimal.py index d61e374b9f9998..13a0dcb77f1267 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -100,9 +100,9 @@ try: from _decimal import * - from _decimal import __version__ - from _decimal import __libmpdec_version__ + from _decimal import __version__ # noqa: F401 + from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: from _pydecimal import * - from _pydecimal import __version__ - from _pydecimal import __libmpdec_version__ + from _pydecimal import __version__ # noqa: F401 + from _pydecimal import __libmpdec_version__ # noqa: F401 diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 1b16441cb60ba7..da0577023cf47d 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -187,7 +187,7 @@ def __hash_new(name, data=b'', **kwargs): try: # OpenSSL's scrypt requires OpenSSL 1.1+ - from _hashlib import scrypt + from _hashlib import scrypt # noqa: F401 except ImportError: pass diff --git a/Lib/lzma.py b/Lib/lzma.py index c1e3d33deb69a1..946066aa0fba56 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -25,7 +25,7 @@ import io import os from _lzma import * -from _lzma import _encode_filter_properties, _decode_filter_properties +from _lzma import _encode_filter_properties, _decode_filter_properties # noqa: F401 import _compression diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index de8a264829dff3..ddcc7e7900999e 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -167,7 +167,7 @@ def allow_connection_pickling(self): ''' # This is undocumented. In previous versions of multiprocessing # its only effect was to make socket objects inheritable on Windows. - from . import connection + from . import connection # noqa: F401 def set_executable(self, executable): '''Sets the path to a python.exe or pythonw.exe binary used to run diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 75dde02d88c533..4f471fbde71ace 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -14,7 +14,7 @@ import atexit import threading # we want threading to install it's # cleanup function before multiprocessing does -from subprocess import _args_from_interpreter_flags +from subprocess import _args_from_interpreter_flags # noqa: F401 from . import process diff --git a/Lib/opcode.py b/Lib/opcode.py index 85e37ff53e577f..85c0834c698ba2 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -12,8 +12,8 @@ import _opcode from _opcode import stack_effect -from _opcode_metadata import (_specializations, _specialized_opmap, opmap, - HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, # noqa: F401 + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) # noqa: F401 EXTENDED_ARG = opmap['EXTENDED_ARG'] opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] diff --git a/Lib/operator.py b/Lib/operator.py index 02ccdaa13ddb31..6d2a762bc95b6d 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -415,7 +415,7 @@ def ixor(a, b): except ImportError: pass else: - from _operator import __doc__ + from _operator import __doc__ # noqa: F401 # All of these "__func__ = func" assignments have to happen after importing # from _operator to make sure they're set to the right function diff --git a/Lib/platform.py b/Lib/platform.py index a4fd2463f15a6c..d6322c9d99d2f3 100644 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -546,7 +546,7 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): warnings._deprecated('java_ver', remove=(3, 15)) # Import the needed APIs try: - import java.lang + import java.lang # noqa: F401 except ImportError: return release, vendor, vminfo, osinfo diff --git a/Lib/pstats.py b/Lib/pstats.py index 2f054bb4011e7f..a174a545456e1a 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -611,7 +611,7 @@ def f8(x): if __name__ == '__main__': import cmd try: - import readline + import readline # noqa: F401 except ImportError: pass diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 278e4846ebb71f..be5cd9a80db710 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -75,9 +75,12 @@ class or function within a module or module in a package. If the from reprlib import Repr from traceback import format_exception_only -from _pyrepl.pager import (get_pager, plain, pipe_pager, +from _pyrepl.pager import (get_pager, pipe_pager, plain_pager, tempfile_pager, tty_pager) +# Expose plain() as pydoc.plain() +from _pyrepl.pager import plain # noqa: F401 + # --------------------------------------------------------- old names diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py index 9c3c294ba448b4..4cb88c96d92715 100644 --- a/Lib/re/_constants.py +++ b/Lib/re/_constants.py @@ -15,7 +15,7 @@ MAGIC = 20230612 -from _sre import MAXREPEAT, MAXGROUPS +from _sre import MAXREPEAT, MAXGROUPS # noqa: F401 # SRE standard exception (access as sre.error) # should this really be here? diff --git a/Lib/site.py b/Lib/site.py index 7eace190f5ab21..9381f6f510eb46 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -486,7 +486,7 @@ def register_readline(): import atexit try: import readline - import rlcompleter + import rlcompleter # noqa: F401 import _pyrepl.readline import _pyrepl.unix_console except ImportError: @@ -603,7 +603,7 @@ def execsitecustomize(): """Run custom site specific code, if available.""" try: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError as exc: if exc.name == 'sitecustomize': pass @@ -623,7 +623,7 @@ def execusercustomize(): """Run custom user specific code, if available.""" try: try: - import usercustomize + import usercustomize # noqa: F401 except ImportError as exc: if exc.name == 'usercustomize': pass diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index b93b84384a0925..d9423c25e34135 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -117,7 +117,7 @@ def main(*args): # No SQL provided; start the REPL. console = SqliteInteractiveConsole(con) try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg="") diff --git a/Lib/struct.py b/Lib/struct.py index d6bba588636498..ff98e8c4cb3f1d 100644 --- a/Lib/struct.py +++ b/Lib/struct.py @@ -11,5 +11,5 @@ ] from _struct import * -from _struct import _clearcache -from _struct import __doc__ +from _struct import _clearcache # noqa: F401 +from _struct import __doc__ # noqa: F401 diff --git a/Lib/symtable.py b/Lib/symtable.py index d6ac1f527ba8ba..f8ba3496439535 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -3,9 +3,9 @@ import _symtable from _symtable import ( USE, - DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, - DEF_PARAM, DEF_TYPE_PARAM, - DEF_FREE_CLASS, + DEF_GLOBAL, # noqa: F401 + DEF_NONLOCAL, DEF_LOCAL, + DEF_PARAM, DEF_TYPE_PARAM, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND, DEF_ANNOT, DEF_COMP_ITER, DEF_COMP_CELL, SCOPE_OFF, SCOPE_MASK, diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index f1f6c911ef17d9..324e5d038aef03 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -57,9 +57,9 @@ def testMultiply(self): from .case import (addModuleCleanup, TestCase, FunctionTestCase, SkipTest, skip, skipIf, skipUnless, expectedFailure, doModuleCleanups, enterModuleContext) -from .suite import BaseTestSuite, TestSuite +from .suite import BaseTestSuite, TestSuite # noqa: F401 from .loader import TestLoader, defaultTestLoader -from .main import TestProgram, main +from .main import TestProgram, main # noqa: F401 from .runner import TextTestRunner, TextTestResult from .signals import installHandler, registerResult, removeResult, removeHandler # IsolatedAsyncioTestCase will be imported lazily. diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index ac6719ce854182..58b0cb574a764a 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -108,7 +108,7 @@ # check for SSL try: - import ssl + import ssl # noqa: F401 except ImportError: _have_ssl = False else: diff --git a/Lib/xml/dom/__init__.py b/Lib/xml/dom/__init__.py index 97cf9a6429993d..dd7fb996afd616 100644 --- a/Lib/xml/dom/__init__.py +++ b/Lib/xml/dom/__init__.py @@ -137,4 +137,4 @@ class UserDataHandler: EMPTY_NAMESPACE = None EMPTY_PREFIX = None -from .domreg import getDOMImplementation, registerDOMImplementation +from .domreg import getDOMImplementation, registerDOMImplementation # noqa: F401 From 2078eb45ca0db495972a20fcaf96df8fcf48451d Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Thu, 13 Jun 2024 16:28:59 +0200 Subject: [PATCH 472/903] gh-120397: Optimize str.count() for single characters (#120398) --- ...-06-12-13-47-25.gh-issue-120397.n-I_cc.rst | 2 ++ Objects/stringlib/fastsearch.h | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst new file mode 100644 index 00000000000000..05c55e8a45eb12 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst @@ -0,0 +1,2 @@ +Improve the througput by up to two times for the :meth:`str.count`, :meth:`bytes.count` and :meth:`bytearray.count` +methods for counting single characters. diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index 309ed1554f4699..05e700b06258f0 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -753,6 +753,22 @@ STRINGLIB(count_char)(const STRINGLIB_CHAR *s, Py_ssize_t n, } +static inline Py_ssize_t +STRINGLIB(count_char_no_maxcount)(const STRINGLIB_CHAR *s, Py_ssize_t n, + const STRINGLIB_CHAR p0) +/* A specialized function of count_char that does not cut off at a maximum. + As a result, the compiler is able to vectorize the loop. */ +{ + Py_ssize_t count = 0; + for (Py_ssize_t i = 0; i < n; i++) { + if (s[i] == p0) { + count++; + } + } + return count; +} + + Py_LOCAL_INLINE(Py_ssize_t) FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, const STRINGLIB_CHAR* p, Py_ssize_t m, @@ -773,6 +789,9 @@ FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, else if (mode == FAST_RSEARCH) return STRINGLIB(rfind_char)(s, n, p[0]); else { + if (maxcount == PY_SSIZE_T_MAX) { + return STRINGLIB(count_char_no_maxcount)(s, n, p[0]); + } return STRINGLIB(count_char)(s, n, p[0], maxcount); } } From 6af190f8d0c5dcb1a875072a30caee2eaf448483 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 13 Jun 2024 19:53:45 +0300 Subject: [PATCH 473/903] gh-120397: Fix typo in NEWS entry (#120455) --- .../2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst index 05c55e8a45eb12..24f046d9d89d51 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-12-13-47-25.gh-issue-120397.n-I_cc.rst @@ -1,2 +1,2 @@ -Improve the througput by up to two times for the :meth:`str.count`, :meth:`bytes.count` and :meth:`bytearray.count` +Improve the throughput by up to two times for the :meth:`str.count`, :meth:`bytes.count` and :meth:`bytearray.count` methods for counting single characters. From c2d810b6d4deeea530648a8d0983e3a2adf6c942 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 13 Jun 2024 18:58:46 +0100 Subject: [PATCH 474/903] GH-119054: Add "Creating files and directories" section to pathlib docs. (#120186) Add dedicated subsection for `pathlib.Path.touch()`, `mkdir()`, `symlink_to()` and `hardlink_to()`. Also note that `open()`, `write_text()` and `write_bytes()` are often used to create files. Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/pathlib.rst | 165 +++++++++++++++++++++------------------- 1 file changed, 86 insertions(+), 79 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index b7ab44706a0160..138e41404dec9c 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1343,6 +1343,92 @@ Reading directories .. versionadded:: 3.12 +Creating files and directories +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. method:: Path.touch(mode=0o666, exist_ok=True) + + Create a file at this given path. If *mode* is given, it is combined + with the process's ``umask`` value to determine the file mode and access + flags. If the file already exists, the function succeeds when *exist_ok* + is true (and its modification time is updated to the current time), + otherwise :exc:`FileExistsError` is raised. + + .. seealso:: + The :meth:`~Path.open`, :meth:`~Path.write_text` and + :meth:`~Path.write_bytes` methods are often used to create files. + + +.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) + + Create a new directory at this given path. If *mode* is given, it is + combined with the process's ``umask`` value to determine the file mode + and access flags. If the path already exists, :exc:`FileExistsError` + is raised. + + If *parents* is true, any missing parents of this path are created + as needed; they are created with the default permissions without taking + *mode* into account (mimicking the POSIX ``mkdir -p`` command). + + If *parents* is false (the default), a missing parent raises + :exc:`FileNotFoundError`. + + If *exist_ok* is false (the default), :exc:`FileExistsError` is + raised if the target directory already exists. + + If *exist_ok* is true, :exc:`FileExistsError` will not be raised unless the given + path already exists in the file system and is not a directory (same + behavior as the POSIX ``mkdir -p`` command). + + .. versionchanged:: 3.5 + The *exist_ok* parameter was added. + + +.. method:: Path.symlink_to(target, target_is_directory=False) + + Make this path a symbolic link pointing to *target*. + + On Windows, a symlink represents either a file or a directory, and does not + morph to the target dynamically. If the target is present, the type of the + symlink will be created to match. Otherwise, the symlink will be created + as a directory if *target_is_directory* is true or a file symlink (the + default) otherwise. On non-Windows platforms, *target_is_directory* is ignored. + + :: + + >>> p = Path('mylink') + >>> p.symlink_to('setup.py') + >>> p.resolve() + PosixPath('/home/antoine/pathlib/setup.py') + >>> p.stat().st_size + 956 + >>> p.lstat().st_size + 8 + + .. note:: + The order of arguments (link, target) is the reverse + of :func:`os.symlink`'s. + + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.symlink` is not + available. In previous versions, :exc:`NotImplementedError` was raised. + + +.. method:: Path.hardlink_to(target) + + Make this path a hard link to the same file as *target*. + + .. note:: + The order of arguments (link, target) is the reverse + of :func:`os.link`'s. + + .. versionadded:: 3.10 + + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.link` is not + available. In previous versions, :exc:`NotImplementedError` was raised. + + Other methods ^^^^^^^^^^^^^ @@ -1426,31 +1512,6 @@ Other methods symbolic link's mode is changed rather than its target's. -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) - - Create a new directory at this given path. If *mode* is given, it is - combined with the process' ``umask`` value to determine the file mode - and access flags. If the path already exists, :exc:`FileExistsError` - is raised. - - If *parents* is true, any missing parents of this path are created - as needed; they are created with the default permissions without taking - *mode* into account (mimicking the POSIX ``mkdir -p`` command). - - If *parents* is false (the default), a missing parent raises - :exc:`FileNotFoundError`. - - If *exist_ok* is false (the default), :exc:`FileExistsError` is - raised if the target directory already exists. - - If *exist_ok* is true, :exc:`FileExistsError` will not be raised unless the given - path already exists in the file system and is not a directory (same - behavior as the POSIX ``mkdir -p`` command). - - .. versionchanged:: 3.5 - The *exist_ok* parameter was added. - - .. method:: Path.owner(*, follow_symlinks=True) Return the name of the user owning the file. :exc:`KeyError` is raised @@ -1572,60 +1633,6 @@ Other methods Remove this directory. The directory must be empty. -.. method:: Path.symlink_to(target, target_is_directory=False) - - Make this path a symbolic link pointing to *target*. - - On Windows, a symlink represents either a file or a directory, and does not - morph to the target dynamically. If the target is present, the type of the - symlink will be created to match. Otherwise, the symlink will be created - as a directory if *target_is_directory* is ``True`` or a file symlink (the - default) otherwise. On non-Windows platforms, *target_is_directory* is ignored. - - :: - - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - >>> p.stat().st_size - 956 - >>> p.lstat().st_size - 8 - - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.symlink`'s. - - .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.symlink` is not - available. In previous versions, :exc:`NotImplementedError` was raised. - - -.. method:: Path.hardlink_to(target) - - Make this path a hard link to the same file as *target*. - - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.link`'s. - - .. versionadded:: 3.10 - - .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.link` is not - available. In previous versions, :exc:`NotImplementedError` was raised. - - -.. method:: Path.touch(mode=0o666, exist_ok=True) - - Create a file at this given path. If *mode* is given, it is combined - with the process' ``umask`` value to determine the file mode and access - flags. If the file already exists, the function succeeds if *exist_ok* - is true (and its modification time is updated to the current time), - otherwise :exc:`FileExistsError` is raised. - - .. method:: Path.unlink(missing_ok=False) Remove this file or symbolic link. If the path points to a directory, From 50a389565aa0b480792ed06a2ab56fb5a72fc2d8 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 14 Jun 2024 03:05:03 +0900 Subject: [PATCH 475/903] gh-117398: Add datetime C-API type check test for subinterpreters (gh-119604) Check if the DateTime C-API type matches the datetime.date type on main and shared/isolated subinterpreters. --- Lib/test/datetimetester.py | 41 ++++++++++++++++++++++++++++++ Lib/test/support/__init__.py | 2 +- Modules/_testcapi/datetime.c | 48 +++++++++++++++++++++++++++++++++--- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 28f75a803b4e04..45188731eed688 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -13,6 +13,7 @@ import re import struct import sys +import textwrap import unittest import warnings @@ -38,6 +39,10 @@ import _testcapi except ImportError: _testcapi = None +try: + import _interpreters +except ModuleNotFoundError: + _interpreters = None # Needed by test_datetime import _strptime @@ -6780,6 +6785,42 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) + def test_type_check_in_subinterp(self): + script = textwrap.dedent(f""" + if {_interpreters is None}: + import _testcapi as module + module.test_datetime_capi() + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + def run(type_checker, obj): + if not type_checker(obj, True): + raise TypeError(f'{{type(obj)}} is not C API type') + + import _datetime + run(module.datetime_check_date, _datetime.date.today()) + run(module.datetime_check_datetime, _datetime.datetime.now()) + run(module.datetime_check_time, _datetime.time(12, 30)) + run(module.datetime_check_delta, _datetime.timedelta(1)) + run(module.datetime_check_tzinfo, _datetime.tzinfo()) + """) + if _interpreters is None: + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + else: + for name in ('isolated', 'legacy'): + with self.subTest(name): + config = _interpreters.new_config(name).__dict__ + ret = support.run_in_subinterp_with_config(script, **config) + self.assertEqual(ret, 0) + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 9e6100d2b89d6e..adc6362e20df00 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1808,7 +1808,7 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): config['gil'] = 'shared' elif gil == 2: config['gil'] = 'own' - else: + elif not isinstance(gil, str): raise NotImplementedError(gil) config = types.SimpleNamespace(**config) return _testinternalcapi.run_in_subinterp_with_config(code, config) diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index b1796039f0d83a..f3d54215e04232 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -22,10 +22,17 @@ test_datetime_capi(PyObject *self, PyObject *args) test_run_counter++; PyDateTime_IMPORT; - if (PyDateTimeAPI) { - Py_RETURN_NONE; + if (PyDateTimeAPI == NULL) { + return NULL; } - return NULL; + // The following C API types need to outlive interpreters, since the + // borrowed references to them can be held by users without being updated. + assert(!PyType_HasFeature(PyDateTimeAPI->DateType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DateTimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DeltaType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TZInfoType, Py_TPFLAGS_HEAPTYPE)); + Py_RETURN_NONE; } /* Functions exposing the C API type checking for testing */ @@ -479,3 +486,38 @@ _PyTestCapi_Init_DateTime(PyObject *mod) } return 0; } + + +/* --------------------------------------------------------------------------- + * Test module for subinterpreters. + */ + +static int +_testcapi_datetime_exec(PyObject *mod) +{ + if (test_datetime_capi(NULL, NULL) == NULL) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot _testcapi_datetime_slots[] = { + {Py_mod_exec, _testcapi_datetime_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static struct PyModuleDef _testcapi_datetime_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_testcapi_datetime", + .m_size = 0, + .m_methods = test_methods, + .m_slots = _testcapi_datetime_slots, +}; + +PyMODINIT_FUNC +PyInit__testcapi_datetime(void) +{ + return PyModuleDef_Init(&_testcapi_datetime_module); +} From 6674c63dc7bb175acc997ddcb799e8dbbafd2968 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 13 Jun 2024 21:01:05 +0200 Subject: [PATCH 476/903] Add codeowner for Makefile.pre.in and Modules/Setup* (#120468) --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8bc40fcb9e8999..1f9047ab97e934 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,8 @@ # Build system configure* @erlend-aasland @corona10 +Makefile.pre.in @erlend-aasland +Modules/Setup* @erlend-aasland # asyncio **/*asyncio* @1st1 @asvetlov @gvanrossum @kumaraditya303 @willingc From a3711afefa7a520b3de01be3b2367cb830d1fc84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 13 Jun 2024 21:03:01 +0200 Subject: [PATCH 477/903] gh-120012: clarify the behaviour of `multiprocessing.Queue.empty` on closed queues. (GH-120102) * improve doc for `multiprocessing.Queue.empty` * add tests for checking emptiness of queues Co-authored-by: Gregory P. Smith --- Doc/library/multiprocessing.rst | 4 +++ Lib/test/_test_multiprocessing.py | 26 +++++++++++++++++++ ...-06-05-12-36-18.gh-issue-120012.f14DbQ.rst | 3 +++ 3 files changed, 33 insertions(+) create mode 100644 Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 49762491bae5f4..426291c5f0743d 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -837,6 +837,8 @@ For an example of the usage of queues for interprocess communication see Return ``True`` if the queue is empty, ``False`` otherwise. Because of multithreading/multiprocessing semantics, this is not reliable. + May raise an :exc:`OSError` on closed queues. (not guaranteed) + .. method:: full() Return ``True`` if the queue is full, ``False`` otherwise. Because of @@ -940,6 +942,8 @@ For an example of the usage of queues for interprocess communication see Return ``True`` if the queue is empty, ``False`` otherwise. + Always raises an :exc:`OSError` if the SimpleQueue is closed. + .. method:: get() Remove and return an item from the queue. diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 301541a666e140..4b3a0645cfc84a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1332,6 +1332,23 @@ def _on_queue_feeder_error(e, obj): self.assertTrue(not_serializable_obj.reduce_was_called) self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called) + def test_closed_queue_empty_exceptions(self): + # Assert that checking the emptiness of an unused closed queue + # does not raise an OSError. The rationale is that q.close() is + # a no-op upon construction and becomes effective once the queue + # has been used (e.g., by calling q.put()). + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.close() # this is a no-op since the feeder thread is None + q.join_thread() # this is also a no-op + self.assertTrue(q.empty()) + + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.put('foo') # make sure that the queue is 'used' + q.close() # close the feeder thread + q.join_thread() # make sure to join the feeder thread + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_closed_queue_put_get_exceptions(self): for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): q.close() @@ -5815,6 +5832,15 @@ def _test_empty(cls, queue, child_can_start, parent_can_continue): finally: parent_can_continue.set() + def test_empty_exceptions(self): + # Assert that checking emptiness of a closed queue raises + # an OSError, independently of whether the queue was used + # or not. This differs from Queue and JoinableQueue. + q = multiprocessing.SimpleQueue() + q.close() # close the pipe + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_empty(self): queue = multiprocessing.SimpleQueue() child_can_start = multiprocessing.Event() diff --git a/Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst b/Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst new file mode 100644 index 00000000000000..2bf0c977b90387 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst @@ -0,0 +1,3 @@ +Clarify the behaviours of :meth:`multiprocessing.Queue.empty` and +:meth:`multiprocessing.SimpleQueue.empty` on closed queues. +Patch by Bénédikt Tran. From d88a1f2e156cd1072119afa91d4f4dc4037c1b21 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 13 Jun 2024 21:25:26 +0100 Subject: [PATCH 478/903] GH-119054: Add "Renaming and deleting" section to pathlib docs. (#120465) Add dedicated subsection for `pathlib.Path.rename()`, `replace()`, `unlink()` and `rmdir()`. --- Doc/library/pathlib.rst | 124 +++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 138e41404dec9c..278851549c6c3b 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1429,6 +1429,70 @@ Creating files and directories available. In previous versions, :exc:`NotImplementedError` was raised. +Renaming and deleting +^^^^^^^^^^^^^^^^^^^^^ + +.. method:: Path.rename(target) + + Rename this file or directory to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. On Unix, if *target* exists + and is a file, it will be replaced silently if the user has permission. + On Windows, if *target* exists, :exc:`FileExistsError` will be raised. + *target* can be either a string or another path object:: + + >>> p = Path('foo') + >>> p.open('w').write('some text') + 9 + >>> target = Path('bar') + >>> p.rename(target) + PosixPath('bar') + >>> target.open().read() + 'some text' + + The target path may be absolute or relative. Relative paths are interpreted + relative to the current working directory, *not* the directory of the + :class:`!Path` object. + + It is implemented in terms of :func:`os.rename` and gives the same guarantees. + + .. versionchanged:: 3.8 + Added return value, return the new :class:`!Path` instance. + + +.. method:: Path.replace(target) + + Rename this file or directory to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. If *target* points to an + existing file or empty directory, it will be unconditionally replaced. + + The target path may be absolute or relative. Relative paths are interpreted + relative to the current working directory, *not* the directory of the + :class:`!Path` object. + + .. versionchanged:: 3.8 + Added return value, return the new :class:`!Path` instance. + + +.. method:: Path.unlink(missing_ok=False) + + Remove this file or symbolic link. If the path points to a directory, + use :func:`Path.rmdir` instead. + + If *missing_ok* is false (the default), :exc:`FileNotFoundError` is + raised if the path does not exist. + + If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be + ignored (same behavior as the POSIX ``rm -f`` command). + + .. versionchanged:: 3.8 + The *missing_ok* parameter was added. + + +.. method:: Path.rmdir() + + Remove this directory. The directory must be empty. + + Other methods ^^^^^^^^^^^^^ @@ -1545,47 +1609,6 @@ Other methods available. In previous versions, :exc:`NotImplementedError` was raised. -.. method:: Path.rename(target) - - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. On Unix, if *target* exists and is a file, - it will be replaced silently if the user has permission. - On Windows, if *target* exists, :exc:`FileExistsError` will be raised. - *target* can be either a string or another path object:: - - >>> p = Path('foo') - >>> p.open('w').write('some text') - 9 - >>> target = Path('bar') - >>> p.rename(target) - PosixPath('bar') - >>> target.open().read() - 'some text' - - The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. - - It is implemented in terms of :func:`os.rename` and gives the same guarantees. - - .. versionchanged:: 3.8 - Added return value, return the new Path instance. - - -.. method:: Path.replace(target) - - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. If *target* points to an existing file or - empty directory, it will be unconditionally replaced. - - The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. - - .. versionchanged:: 3.8 - Added return value, return the new Path instance. - - .. method:: Path.absolute() Make the path absolute, without normalization or resolving symlinks. @@ -1628,25 +1651,6 @@ Other methods strict mode, and no exception is raised in non-strict mode. In previous versions, :exc:`RuntimeError` is raised no matter the value of *strict*. -.. method:: Path.rmdir() - - Remove this directory. The directory must be empty. - - -.. method:: Path.unlink(missing_ok=False) - - Remove this file or symbolic link. If the path points to a directory, - use :func:`Path.rmdir` instead. - - If *missing_ok* is false (the default), :exc:`FileNotFoundError` is - raised if the path does not exist. - - If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be - ignored (same behavior as the POSIX ``rm -f`` command). - - .. versionchanged:: 3.8 - The *missing_ok* parameter was added. - .. _pathlib-pattern-language: From 42351c3b9a357ec67135b30ed41f59e6f306ac52 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 13 Jun 2024 22:16:40 +0100 Subject: [PATCH 479/903] gh-114053: Fix bad interaction of PEP 695, PEP 563 and `inspect.get_annotations` (#120270) --- Lib/inspect.py | 8 +- .../inspect_stringized_annotations_pep695.py | 72 ++++++++++++ Lib/test/test_inspect/test_inspect.py | 103 ++++++++++++++++++ ...-06-08-15-15-29.gh-issue-114053.WQLAFG.rst | 4 + 4 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_inspect/inspect_stringized_annotations_pep695.py create mode 100644 Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 5570a43ebfea19..11544b8d0d4932 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -274,7 +274,13 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False): if globals is None: globals = obj_globals if locals is None: - locals = obj_locals + locals = obj_locals or {} + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + locals = {param.__name__: param for param in type_params} | locals return_value = {key: value if not isinstance(value, str) else eval(value, globals, locals) diff --git a/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py new file mode 100644 index 00000000000000..723822f8eaa92d --- /dev/null +++ b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py @@ -0,0 +1,72 @@ +from __future__ import annotations +from typing import Callable, Unpack + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +Eggs = int +Spam = str + + +class C[Eggs, **Spam]: + x: Eggs + y: Spam + + +def generic_function[T, *Ts, **P]( + x: T, *y: Unpack[Ts], z: P.args, zz: P.kwargs +) -> None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +def nested(): + from types import SimpleNamespace + from inspect import get_annotations + + Eggs = bytes + Spam = memoryview + + + class E[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + return SimpleNamespace( + E=E, + E_annotations=get_annotations(E, eval_str=True), + E_meth_annotations=get_annotations(E.generic_method, eval_str=True), + generic_func=generic_function, + generic_func_annotations=get_annotations(generic_function, eval_str=True) + ) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 0a4fa9343f15e0..140efac530afb2 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -22,6 +22,7 @@ import types import tempfile import textwrap +from typing import Unpack import unicodedata import unittest import unittest.mock @@ -47,6 +48,7 @@ from test.test_inspect import inspect_stock_annotations from test.test_inspect import inspect_stringized_annotations from test.test_inspect import inspect_stringized_annotations_2 +from test.test_inspect import inspect_stringized_annotations_pep695 # Functions tested in this suite: @@ -1692,6 +1694,107 @@ def wrapper(a, b): self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations), {'x': 'mytype'}) self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), {'x': int}) + def test_pep695_generic_class_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + A_annotations = inspect.get_annotations(ann_module695.A, eval_str=True) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(A_annotations["x"], A_type_params[0]) + self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + B_annotations = inspect.get_annotations( + inspect_stringized_annotations_pep695.B, eval_str=True + ) + self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): + ann_module695 = inspect_stringized_annotations_pep695 + C_annotations = inspect.get_annotations(ann_module695.C, eval_str=True) + self.assertEqual( + set(C_annotations.values()), + set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_func_annotations = inspect.get_annotations( + ann_module695.generic_function, eval_str=True + ) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(generic_func_annotations["x"], func_t_params[0]) + self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) + self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) + self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set( + inspect.get_annotations( + inspect_stringized_annotations_pep695.generic_function_2, + eval_str=True + ).values() + ), + set( + inspect_stringized_annotations_pep695.generic_function_2.__type_params__ + ) + ) + + def test_pep_695_generic_method_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_method_annotations = inspect.get_annotations( + ann_module695.D.generic_method, eval_str=True + ) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + generic_method_annotations, + {"x": params["Foo"], "y": params["Bar"], "return": None} + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set( + inspect.get_annotations( + inspect_stringized_annotations_pep695.D.generic_method_2, + eval_str=True + ).values() + ), + set( + inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ + ) + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = inspect_stringized_annotations_pep695.nested() + + self.assertEqual( + set(results.E_annotations.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.E_meth_annotations.values()), + set(results.E.generic_method.__type_params__) + ) + self.assertNotEqual( + set(results.E_meth_annotations.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.E_meth_annotations.values()).intersection(results.E.__type_params__), + set() + ) + + self.assertEqual( + set(results.generic_func_annotations.values()), + set(results.generic_func.__type_params__) + ) + class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): diff --git a/Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst b/Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst new file mode 100644 index 00000000000000..be49577a712867 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst @@ -0,0 +1,4 @@ +Fix erroneous :exc:`NameError` when calling :func:`inspect.get_annotations` +with ``eval_str=True``` on a class that made use of :pep:`695` type +parameters in a module that had ``from __future__ import annotations`` at +the top of the file. Patch by Alex Waygood. From 41554ef0e0925695544d96a7bc49af1428d6bb6b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Jun 2024 10:21:35 -0500 Subject: [PATCH 480/903] Stronger tests for the statistics kernel formulas (gh-120506) --- Lib/test/test_statistics.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 0b28459f03d86a..c374c947e02a6b 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2434,18 +2434,22 @@ def integrate(func, low, high, steps=10_000): data.append(100) self.assertGreater(f_hat(100), 0.0) - def test_kde_kernel_invcdfs(self): + def test_kde_kernel_specs(self): + # White-box test for the kernel formulas in isolation from + # their downstream use in kde() and kde_random() kernel_specs = statistics._kernel_specs - kde = statistics.kde # Verify that cdf / invcdf will round trip xarr = [i/100 for i in range(-100, 101)] + parr = [i/1000 + 5/10000 for i in range(1000)] for kernel, spec in kernel_specs.items(): + cdf = spec['cdf'] invcdf = spec['invcdf'] with self.subTest(kernel=kernel): - cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) for x in xarr: self.assertAlmostEqual(invcdf(cdf(x)), x, places=6) + for p in parr: + self.assertAlmostEqual(cdf(invcdf(p)), p, places=11) @support.requires_resource('cpu') def test_kde_random(self): From 27419f1fce05a18384e6fb3b8ad59b7f532821e6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Jun 2024 11:00:46 -0500 Subject: [PATCH 481/903] Update tests for the itertools docs rough equivalents (#120509) --- Lib/test/test_itertools.py | 333 +++++++++++++++++++++++++++++++++++-- 1 file changed, 315 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 53b8064c3cfe82..5fd6ecf37427f7 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1587,27 +1587,169 @@ def batched_recipe(iterable, n): self.assertEqual(r1, r2) self.assertEqual(e1, e2) + + def test_groupby_recipe(self): + + # Begin groupby() recipe ####################################### + + def groupby(iterable, key=None): + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D + + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: + return + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass + + # End groupby() recipe ######################################### + + # Check whether it accepts arguments correctly + self.assertEqual([], list(groupby([]))) + self.assertEqual([], list(groupby([], key=id))) + self.assertRaises(TypeError, list, groupby('abc', [])) + if False: + # Test not applicable to the recipe + self.assertRaises(TypeError, list, groupby('abc', None)) + self.assertRaises(TypeError, groupby, 'abc', lambda x:x, 10) + + # Check normal input + s = [(0, 10, 20), (0, 11,21), (0,12,21), (1,13,21), (1,14,22), + (2,15,22), (3,16,23), (3,17,23)] + dup = [] + for k, g in groupby(s, lambda r:r[0]): + for elem in g: + self.assertEqual(k, elem[0]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check nested case + dup = [] + for k, g in groupby(s, testR): + for ik, ig in groupby(g, testR2): + for elem in ig: + self.assertEqual(k, elem[0]) + self.assertEqual(ik, elem[2]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check case where inner iterator is not used + keys = [k for k, g in groupby(s, testR)] + expectedkeys = set([r[0] for r in s]) + self.assertEqual(set(keys), expectedkeys) + self.assertEqual(len(keys), len(expectedkeys)) + + # Check case where inner iterator is used after advancing the groupby + # iterator + s = list(zip('AABBBAAAA', range(9))) + it = groupby(s, testR) + _, g1 = next(it) + _, g2 = next(it) + _, g3 = next(it) + self.assertEqual(list(g1), []) + self.assertEqual(list(g2), []) + self.assertEqual(next(g3), ('A', 5)) + list(it) # exhaust the groupby iterator + self.assertEqual(list(g3), []) + + # Exercise pipes and filters style + s = 'abracadabra' + # sort s | uniq + r = [k for k, g in groupby(sorted(s))] + self.assertEqual(r, ['a', 'b', 'c', 'd', 'r']) + # sort s | uniq -d + r = [k for k, g in groupby(sorted(s)) if list(islice(g,1,2))] + self.assertEqual(r, ['a', 'b', 'r']) + # sort s | uniq -c + r = [(len(list(g)), k) for k, g in groupby(sorted(s))] + self.assertEqual(r, [(5, 'a'), (2, 'b'), (1, 'c'), (1, 'd'), (2, 'r')]) + # sort s | uniq -c | sort -rn | head -3 + r = sorted([(len(list(g)) , k) for k, g in groupby(sorted(s))], reverse=True)[:3] + self.assertEqual(r, [(5, 'a'), (2, 'r'), (2, 'b')]) + + # iter.__next__ failure + class ExpectedError(Exception): + pass + def delayed_raise(n=0): + for i in range(n): + yield 'yo' + raise ExpectedError + def gulp(iterable, keyp=None, func=list): + return [func(g) for k, g in groupby(iterable, keyp)] + + # iter.__next__ failure on outer object + self.assertRaises(ExpectedError, gulp, delayed_raise(0)) + # iter.__next__ failure on inner object + self.assertRaises(ExpectedError, gulp, delayed_raise(1)) + + # __eq__ failure + class DummyCmp: + def __eq__(self, dst): + raise ExpectedError + s = [DummyCmp(), DummyCmp(), None] + + # __eq__ failure on outer object + self.assertRaises(ExpectedError, gulp, s, func=id) + # __eq__ failure on inner object + self.assertRaises(ExpectedError, gulp, s) + + # keyfunc failure + def keyfunc(obj): + if keyfunc.skip > 0: + keyfunc.skip -= 1 + return obj + else: + raise ExpectedError + + # keyfunc failure on outer object + keyfunc.skip = 0 + self.assertRaises(ExpectedError, gulp, [None], keyfunc) + keyfunc.skip = 1 + self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + + @staticmethod def islice(iterable, *args): + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step def test_islice_recipe(self): self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) @@ -1627,6 +1769,161 @@ def test_islice_recipe(self): self.assertEqual(next(c), 3) + def test_tee_recipe(self): + + # Begin tee() recipe ########################################### + + def tee(iterable, n=2): + iterator = iter(iterable) + shared_link = [None, None] + return tuple(_tee(iterator, shared_link) for _ in range(n)) + + def _tee(iterator, link): + try: + while True: + if link[1] is None: + link[0] = next(iterator) + link[1] = [None, None] + value, link = link + yield value + except StopIteration: + return + + # End tee() recipe ############################################# + + n = 200 + + a, b = tee([]) # test empty iterator + self.assertEqual(list(a), []) + self.assertEqual(list(b), []) + + a, b = tee(irange(n)) # test 100% interleaved + self.assertEqual(lzip(a,b), lzip(range(n), range(n))) + + a, b = tee(irange(n)) # test 0% interleaved + self.assertEqual(list(a), list(range(n))) + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of leading iterator + for i in range(100): + self.assertEqual(next(a), i) + del a + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of trailing iterator + for i in range(100): + self.assertEqual(next(a), i) + del b + self.assertEqual(list(a), list(range(100, n))) + + for j in range(5): # test randomly interleaved + order = [0]*n + [1]*n + random.shuffle(order) + lists = ([], []) + its = tee(irange(n)) + for i in order: + value = next(its[i]) + lists[i].append(value) + self.assertEqual(lists[0], list(range(n))) + self.assertEqual(lists[1], list(range(n))) + + # test argument format checking + self.assertRaises(TypeError, tee) + self.assertRaises(TypeError, tee, 3) + self.assertRaises(TypeError, tee, [1,2], 'x') + self.assertRaises(TypeError, tee, [1,2], 3, 'x') + + # Tests not applicable to the tee() recipe + if False: + # tee object should be instantiable + a, b = tee('abc') + c = type(a)('def') + self.assertEqual(list(c), list('def')) + + # test long-lagged and multi-way split + a, b, c = tee(range(2000), 3) + for i in range(100): + self.assertEqual(next(a), i) + self.assertEqual(list(b), list(range(2000))) + self.assertEqual([next(c), next(c)], list(range(2))) + self.assertEqual(list(a), list(range(100,2000))) + self.assertEqual(list(c), list(range(2,2000))) + + # Tests not applicable to the tee() recipe + if False: + # test invalid values of n + self.assertRaises(TypeError, tee, 'abc', 'invalid') + self.assertRaises(ValueError, tee, [], -1) + + for n in range(5): + result = tee('abc', n) + self.assertEqual(type(result), tuple) + self.assertEqual(len(result), n) + self.assertEqual([list(x) for x in result], [list('abc')]*n) + + + # Tests not applicable to the tee() recipe + if False: + # tee pass-through to copyable iterator + a, b = tee('abc') + c, d = tee(a) + self.assertTrue(a is c) + + # test tee_new + t1, t2 = tee('abc') + tnew = type(t1) + self.assertRaises(TypeError, tnew) + self.assertRaises(TypeError, tnew, 10) + t3 = tnew(t1) + self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) + + # test that tee objects are weak referencable + a, b = tee(range(10)) + p = weakref.proxy(a) + self.assertEqual(getattr(p, '__class__'), type(b)) + del a + gc.collect() # For PyPy or other GCs. + self.assertRaises(ReferenceError, getattr, p, '__class__') + + ans = list('abc') + long_ans = list(range(10000)) + + # Tests not applicable to the tee() recipe + if False: + # check copy + a, b = tee('abc') + self.assertEqual(list(copy.copy(a)), ans) + self.assertEqual(list(copy.copy(b)), ans) + a, b = tee(list(range(10000))) + self.assertEqual(list(copy.copy(a)), long_ans) + self.assertEqual(list(copy.copy(b)), long_ans) + + # check partially consumed copy + a, b = tee('abc') + take(2, a) + take(1, b) + self.assertEqual(list(copy.copy(a)), ans[2:]) + self.assertEqual(list(copy.copy(b)), ans[1:]) + self.assertEqual(list(a), ans[2:]) + self.assertEqual(list(b), ans[1:]) + a, b = tee(range(10000)) + take(100, a) + take(60, b) + self.assertEqual(list(copy.copy(a)), long_ans[100:]) + self.assertEqual(list(copy.copy(b)), long_ans[60:]) + self.assertEqual(list(a), long_ans[100:]) + self.assertEqual(list(b), long_ans[60:]) + + # Issue 13454: Crash when deleting backward iterator from tee() + forward, backward = tee(repeat(None, 2000)) # 20000000 + try: + any(forward) # exhaust the iterator + del backward + except: + del forward, backward + raise + + class TestGC(unittest.TestCase): def makecycle(self, iterator, container): From 2bacc2343c24c49292dea3461f6b7664fc2d33e2 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 15 Jun 2024 00:10:18 +0800 Subject: [PATCH 482/903] gh-117657: Add TSAN suppression for set_default_allocator_unlocked (#120500) Add TSAN suppression for set_default_allocator_unlocked --- Tools/tsan/suppressions_free_threading.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 05ceaf438b6353..4c8b0b8abd2963 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -63,6 +63,8 @@ race_top:tstate_is_freed race_top:type_modified_unlocked race_top:write_thread_id race_top:PyThreadState_Clear +# Only seen on macOS, sample: https://gist.github.com/aisk/dda53f5d494a4556c35dde1fce03259c +race_top:set_default_allocator_unlocked # https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40 thread:pthread_create From 7c38097add9cc24e9f68414cd3e5e1b6cbe38a17 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 14 Jun 2024 17:15:49 +0100 Subject: [PATCH 483/903] GH-73991: Add `pathlib.Path.copy()` (#119058) Add a `Path.copy()` method that copies the content of one file to another. This method is similar to `shutil.copyfile()` but differs in the following ways: - Uses `fcntl.FICLONE` where available (see GH-81338) - Uses `os.copy_file_range` where available (see GH-81340) - Uses `_winapi.CopyFile2` where available, even though this copies more metadata than the other implementations. This makes `WindowsPath.copy()` more similar to `shutil.copy2()`. The method is presently _less_ specified than the `shutil` functions to allow OS-specific optimizations that might copy more or less metadata. Incorporates code from GH-81338 and GH-93152. Co-authored-by: Eryk Sun --- Doc/library/pathlib.rst | 18 ++- Doc/whatsnew/3.14.rst | 7 + Lib/pathlib/_abc.py | 30 ++++ Lib/pathlib/_local.py | 16 ++ Lib/pathlib/_os.py | 138 ++++++++++++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 62 ++++++++ ...4-05-15-01-36-08.gh-issue-73991.CGknDf.rst | 2 + 7 files changed, 271 insertions(+), 2 deletions(-) create mode 100644 Lib/pathlib/_os.py create mode 100644 Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 278851549c6c3b..c8a3272d7bab4c 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1429,8 +1429,22 @@ Creating files and directories available. In previous versions, :exc:`NotImplementedError` was raised. -Renaming and deleting -^^^^^^^^^^^^^^^^^^^^^ +Copying, renaming and deleting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. method:: Path.copy(target) + + Copy the contents of this file to the *target* file. If *target* specifies + a file that already exists, it will be replaced. + + .. note:: + This method uses operating system functionality to copy file content + efficiently. The OS might also copy some metadata, such as file + permissions. After the copy is complete, users may wish to call + :meth:`Path.chmod` to set the permissions of the target file. + + .. versionadded:: 3.14 + .. method:: Path.rename(target) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b357553735e8bb..a102af13a08362 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -100,6 +100,13 @@ os by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) +pathlib +------- + +* Add :meth:`pathlib.Path.copy`, which copies the content of one file to + another, like :func:`shutil.copyfile`. + (Contributed by Barney Gale in :gh:`73991`.) + symtable -------- diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index ecea8e88d1a2e3..586145ead384ea 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -16,6 +16,7 @@ import posixpath from glob import _GlobberBase, _no_recurse_symlinks from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from ._os import copyfileobj __all__ = ["UnsupportedOperation"] @@ -563,6 +564,15 @@ def samefile(self, other_path): return (st.st_ino == other_st.st_ino and st.st_dev == other_st.st_dev) + def _samefile_safe(self, other_path): + """ + Like samefile(), but returns False rather than raising OSError. + """ + try: + return self.samefile(other_path) + except (OSError, ValueError): + return False + def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ @@ -780,6 +790,26 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ raise UnsupportedOperation(self._unsupported_msg('mkdir()')) + def copy(self, target): + """ + Copy the contents of this file to the given target. + """ + if not isinstance(target, PathBase): + target = self.with_segments(target) + if self._samefile_safe(target): + raise OSError(f"{self!r} and {target!r} are the same file") + with self.open('rb') as source_f: + try: + with target.open('wb') as target_f: + copyfileobj(source_f, target_f) + except IsADirectoryError as e: + if not target.exists(): + # Raise a less confusing exception. + raise FileNotFoundError( + f'Directory does not exist: {target}') from e + else: + raise + def rename(self, target): """ Rename this path to the target path. diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 473fd525768b50..cffed10dbd1207 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -18,6 +18,7 @@ grp = None from ._abc import UnsupportedOperation, PurePathBase, PathBase +from ._os import copyfile __all__ = [ @@ -780,6 +781,21 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): if not exist_ok or not self.is_dir(): raise + if copyfile: + def copy(self, target): + """ + Copy the contents of this file to the given target. + """ + try: + target = os.fspath(target) + except TypeError: + if isinstance(target, PathBase): + # Target is an instance of PathBase but not os.PathLike. + # Use generic implementation from PathBase. + return PathBase.copy(self, target) + raise + copyfile(os.fspath(self), target) + def chmod(self, mode, *, follow_symlinks=True): """ Change the permissions of the path, like os.chmod(). diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py new file mode 100644 index 00000000000000..1771d54e4167c1 --- /dev/null +++ b/Lib/pathlib/_os.py @@ -0,0 +1,138 @@ +""" +Low-level OS functionality wrappers used by pathlib. +""" + +from errno import EBADF, EOPNOTSUPP, ETXTBSY, EXDEV +import os +import sys +try: + import fcntl +except ImportError: + fcntl = None +try: + import posix +except ImportError: + posix = None +try: + import _winapi +except ImportError: + _winapi = None + + +def get_copy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + + +if fcntl and hasattr(fcntl, 'FICLONE'): + def clonefd(source_fd, target_fd): + """ + Perform a lightweight copy of two files, where the data blocks are + copied only when modified. This is known as Copy on Write (CoW), + instantaneous copy or reflink. + """ + fcntl.ioctl(target_fd, fcntl.FICLONE, source_fd) +else: + clonefd = None + + +if posix and hasattr(posix, '_fcopyfile'): + def copyfd(source_fd, target_fd): + """ + Copy a regular file content using high-performance fcopyfile(3) + syscall (macOS). + """ + posix._fcopyfile(source_fd, target_fd, posix._COPYFILE_DATA) +elif hasattr(os, 'copy_file_range'): + def copyfd(source_fd, target_fd): + """ + Copy data from one regular mmap-like fd to another by using a + high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side + copy. + This should work on Linux >= 4.5 only. + """ + blocksize = get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.copy_file_range(source_fd, target_fd, blocksize, + offset_dst=offset) + if sent == 0: + break # EOF + offset += sent +elif hasattr(os, 'sendfile'): + def copyfd(source_fd, target_fd): + """Copy data from one regular mmap-like fd to another by using + high-performance sendfile(2) syscall. + This should work on Linux >= 2.6.33 only. + """ + blocksize = get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.sendfile(target_fd, source_fd, offset, blocksize) + if sent == 0: + break # EOF + offset += sent +else: + copyfd = None + + +if _winapi and hasattr(_winapi, 'CopyFile2'): + def copyfile(source, target): + """ + Copy from one file to another using CopyFile2 (Windows only). + """ + _winapi.CopyFile2(source, target, 0) +else: + copyfile = None + + +def copyfileobj(source_f, target_f): + """ + Copy data from file-like object source_f to file-like object target_f. + """ + try: + source_fd = source_f.fileno() + target_fd = target_f.fileno() + except Exception: + pass # Fall through to generic code. + else: + try: + # Use OS copy-on-write where available. + if clonefd: + try: + clonefd(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EBADF, EOPNOTSUPP, ETXTBSY, EXDEV): + raise err + + # Use OS copy where available. + if copyfd: + copyfd(source_fd, target_fd) + return + except OSError as err: + # Produce more useful error messages. + err.filename = source_f.name + err.filename2 = target_f.name + raise err + + # Last resort: copy with fileobj read() and write(). + read_source = source_f.read + write_target = target_f.write + while buf := read_source(1024 * 1024): + write_target(buf) diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 57cc1612c03468..fd71284159d5c0 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1696,6 +1696,68 @@ def test_write_text_with_newlines(self): self.assertEqual((p / 'fileA').read_bytes(), b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') + def test_copy_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_directory(self): + base = self.cls(self.base) + source = base / 'dirA' + target = base / 'copyA' + with self.assertRaises(OSError): + source.copy(target) + + @needs_symlinks + def test_copy_symlink(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + source.copy(target) + self.assertTrue(target.exists()) + self.assertFalse(target.is_symlink()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_to_existing_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirB' / 'fileB' + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_to_existing_directory(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirA' + with self.assertRaises(OSError): + source.copy(target) + + @needs_symlinks + def test_copy_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + source.copy(target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + def test_copy_empty(self): + base = self.cls(self.base) + source = base / 'empty' + target = base / 'copyA' + source.write_bytes(b'') + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(target.read_bytes(), b'') + def test_iterdir(self): P = self.cls p = P(self.base) diff --git a/Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst b/Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst new file mode 100644 index 00000000000000..c2953c65b2720f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst @@ -0,0 +1,2 @@ +Add :meth:`pathlib.Path.copy`, which copies the content of one file to another, +like :func:`shutil.copyfile`. From 7fadfd82ebf6ea90b38cb3f2a046a51f8601a205 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 14 Jun 2024 20:25:35 +0300 Subject: [PATCH 484/903] gh-120361: Add `nonmember` test with enum flags inside to `test_enum` (GH-120364) * gh-120361: Add `nonmember` test with enum flags inside to `test_enum` --- Doc/library/enum.rst | 2 +- Lib/test/test_enum.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 8c604c2347a547..9cf94e342dad28 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -527,7 +527,7 @@ Data Types ``Flag`` is the same as :class:`Enum`, but its members support the bitwise operators ``&`` (*AND*), ``|`` (*OR*), ``^`` (*XOR*), and ``~`` (*INVERT*); - the results of those operators are members of the enumeration. + the results of those operations are (aliases of) members of the enumeration. .. method:: __contains__(self, value) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 529dfc62eff680..99fd16ba361e6f 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -1495,6 +1495,27 @@ class SpamEnum(Enum): spam = nonmember(SpamEnumIsInner) self.assertTrue(SpamEnum.spam is SpamEnumIsInner) + def test_using_members_as_nonmember(self): + class Example(Flag): + A = 1 + B = 2 + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + + class Example(Flag): + A = auto() + B = auto() + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + def test_nested_classes_in_enum_with_member(self): """Support locally-defined nested classes.""" class Outer(Enum): From ed60ab5fab6d187068cb3e0f0d4192ebf3a228b7 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 14 Jun 2024 11:25:23 -0700 Subject: [PATCH 485/903] gh-119824: Print stack entry when user input is needed (#119882) Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Doc/library/pdb.rst | 10 +++- Lib/pdb.py | 50 +++++++++++++++---- Lib/test/test_pdb.py | 50 ++++++++++++++++--- ...-05-31-21-17-43.gh-issue-119824.CQlxWV.rst | 1 + 4 files changed, 90 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index f6085171dccb38..b1e9392ecfd927 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -321,11 +321,17 @@ can be overridden by the local file. argument must be an identifier, ``help exec`` must be entered to get help on the ``!`` command. -.. pdbcommand:: w(here) +.. pdbcommand:: w(here) [count] - Print a stack trace, with the most recent frame at the bottom. An arrow (``>``) + Print a stack trace, with the most recent frame at the bottom. if *count* + is 0, print the current frame entry. If *count* is negative, print the least + recent - *count* frames. If *count* is positive, print the most recent + *count* frames. An arrow (``>``) indicates the current frame, which determines the context of most commands. + .. versionchanged:: 3.14 + *count* argument is added. + .. pdbcommand:: d(own) [count] Move the current frame *count* (default one) levels down in the stack trace diff --git a/Lib/pdb.py b/Lib/pdb.py index ba84a29aa2f669..ddbfb9d2bb6244 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -603,10 +603,18 @@ def interaction(self, frame, tb_or_exc): assert tb is not None, "main exception must have a traceback" with self._hold_exceptions(_chained_exceptions): self.setup(frame, tb) - # if we have more commands to process, do not show the stack entry - if not self.cmdqueue: + # We should print the stack entry if and only if the user input + # is expected, and we should print it right before the user input. + # If self.cmdqueue is not empty, we append a "w 0" command to the + # queue, which is equivalent to print_stack_entry + if self.cmdqueue: + self.cmdqueue.append('w 0') + else: self.print_stack_entry(self.stack[self.curindex]) self._cmdloop() + # If "w 0" is not used, pop it out + if self.cmdqueue and self.cmdqueue[-1] == 'w 0': + self.cmdqueue.pop() self.forget() def displayhook(self, obj): @@ -1401,16 +1409,24 @@ def do_clear(self, arg): complete_cl = _complete_location def do_where(self, arg): - """w(here) + """w(here) [count] - Print a stack trace, with the most recent frame at the bottom. + Print a stack trace. If count is not specified, print the full stack. + If count is 0, print the current frame entry. If count is positive, + print count entries from the most recent frame. If count is negative, + print -count entries from the least recent frame. An arrow indicates the "current frame", which determines the context of most commands. 'bt' is an alias for this command. """ - if arg: - self._print_invalid_arg(arg) - return - self.print_stack_trace() + if not arg: + count = None + else: + try: + count = int(arg) + except ValueError: + self.error('Invalid count (%s)' % arg) + return + self.print_stack_trace(count) do_w = do_where do_bt = do_where @@ -2065,10 +2081,22 @@ def complete_unalias(self, text, line, begidx, endidx): # It is also consistent with the up/down commands (which are # compatible with dbx and gdb: up moves towards 'main()' # and down moves towards the most recent stack frame). - - def print_stack_trace(self): + # * if count is None, prints the full stack + # * if count = 0, prints the current frame entry + # * if count < 0, prints -count least recent frame entries + # * if count > 0, prints count most recent frame entries + + def print_stack_trace(self, count=None): + if count is None: + stack_to_print = self.stack + elif count == 0: + stack_to_print = [self.stack[self.curindex]] + elif count < 0: + stack_to_print = self.stack[:-count] + else: + stack_to_print = self.stack[-count:] try: - for frame_lineno in self.stack: + for frame_lineno in stack_to_print: self.print_stack_entry(frame_lineno) except KeyboardInterrupt: pass diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index cf69bc415c9b69..5edf68dc3b429b 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -781,7 +781,7 @@ def test_pdb_where_command(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() >>> def f(): - ... g(); + ... g() >>> def test_function(): ... f() @@ -789,8 +789,13 @@ def test_pdb_where_command(): >>> with PdbTestInput([ # doctest: +ELLIPSIS ... 'w', ... 'where', + ... 'w 1', + ... 'w invalid', ... 'u', ... 'w', + ... 'w 0', + ... 'w 100', + ... 'w -100', ... 'continue', ... ]): ... test_function() @@ -798,35 +803,63 @@ def test_pdb_where_command(): -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) w ... - (8)() + (13)() -> test_function() (2)test_function() -> f() (2)f() - -> g(); + -> g() > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) where ... - (8)() + (13)() -> test_function() (2)test_function() -> f() (2)f() - -> g(); + -> g() > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w 1 + > (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w invalid + *** Invalid count (invalid) (Pdb) u > (2)f() - -> g(); + -> g() (Pdb) w ... - (8)() + (13)() + -> test_function() + (2)test_function() + -> f() + > (2)f() + -> g() + (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w 0 + > (2)f() + -> g() + (Pdb) w 100 + ... + (13)() -> test_function() (2)test_function() -> f() > (2)f() - -> g(); + -> g() + (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w -100 + ... + (13)() + -> test_function() + (2)test_function() + -> f() + > (2)f() + -> g() (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue @@ -3179,6 +3212,7 @@ def test_pdbrc_basic(self): stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) self.assertNotIn("SyntaxError", stdout) self.assertIn("a+8=9", stdout) + self.assertIn("-> b = 2", stdout) def test_pdbrc_empty_line(self): """Test that empty lines in .pdbrc are ignored.""" diff --git a/Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst b/Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst new file mode 100644 index 00000000000000..fd6d8d79a9d157 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst @@ -0,0 +1 @@ +Print stack entry in :mod:`pdb` when and only when user input is needed. From 05df063ad80becc1ba6bd07d67b55b5965f32375 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Jun 2024 20:39:50 +0200 Subject: [PATCH 486/903] gh-120417: Fix "imported but unused" linter warnings (#120461) Add __all__ to the following modules: importlib.machinery, importlib.util and xml.sax. Add also "# noqa: F401" in collections.abc, subprocess and xml.sax. * Sort __all__; remove collections.abc.__all__; remove private names * Add tests --- Lib/collections/abc.py | 4 +-- Lib/importlib/machinery.py | 8 ++++++ Lib/importlib/util.py | 6 +++++ Lib/subprocess.py | 2 +- Lib/test/test_importlib/test_api.py | 40 +++++++++++++++++++++++++++++ Lib/test/test_sax.py | 18 ++++++++++++- Lib/xml/sax/__init__.py | 14 +++++++--- 7 files changed, 84 insertions(+), 8 deletions(-) diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index 86ca8b8a8414b3..bff76291634604 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,3 @@ from _collections_abc import * -from _collections_abc import __all__ -from _collections_abc import _CallableGenericAlias +from _collections_abc import __all__ # noqa: F401 +from _collections_abc import _CallableGenericAlias # noqa: F401 diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index fbd30b159fb752..6e294d59bfdcb9 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -19,3 +19,11 @@ def all_suffixes(): """Returns a list of all recognized module suffixes for this process""" return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES + + +__all__ = ['AppleFrameworkLoader', 'BYTECODE_SUFFIXES', 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', 'FileFinder', 'FrozenImporter', 'ModuleSpec', + 'NamespaceLoader', 'OPTIMIZED_BYTECODE_SUFFIXES', 'PathFinder', + 'SOURCE_SUFFIXES', 'SourceFileLoader', 'SourcelessFileLoader', + 'WindowsRegistryFinder', 'all_suffixes'] diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index c94a148e4c50e0..7243d052cc27f3 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -270,3 +270,9 @@ def exec_module(self, module): loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule + + +__all__ = ['LazyLoader', 'Loader', 'MAGIC_NUMBER', + 'cache_from_source', 'decode_source', 'find_spec', + 'module_from_spec', 'resolve_name', 'source_from_cache', + 'source_hash', 'spec_from_file_location', 'spec_from_loader'] diff --git a/Lib/subprocess.py b/Lib/subprocess.py index b2dcb1454c139e..bc08878db313df 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -79,7 +79,7 @@ if _mswindows: import _winapi - from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, # noqa: F401 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 2a35f3dcb7210c..973237c0791a3e 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -6,6 +6,7 @@ import os.path import sys +from test import support from test.support import import_helper from test.support import os_helper import types @@ -437,5 +438,44 @@ def test_everyone_has___spec__(self): ) = test_util.test_both(StartupTests, machinery=machinery) +class TestModuleAll(unittest.TestCase): + def test_machinery(self): + extra = ( + # from importlib._bootstrap and importlib._bootstrap_external + 'AppleFrameworkLoader', + 'BYTECODE_SUFFIXES', + 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', + 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', + 'FileFinder', + 'FrozenImporter', + 'ModuleSpec', + 'NamespaceLoader', + 'OPTIMIZED_BYTECODE_SUFFIXES', + 'PathFinder', + 'SOURCE_SUFFIXES', + 'SourceFileLoader', + 'SourcelessFileLoader', + 'WindowsRegistryFinder', + ) + support.check__all__(self, machinery['Source'], extra=extra) + + def test_util(self): + extra = ( + # from importlib.abc, importlib._bootstrap + # and importlib._bootstrap_external + 'Loader', + 'MAGIC_NUMBER', + 'cache_from_source', + 'decode_source', + 'module_from_spec', + 'source_from_cache', + 'spec_from_file_location', + 'spec_from_loader', + ) + support.check__all__(self, util['Source'], extra=extra) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 9b3014a94a081e..0d0f86c145b499 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -16,6 +16,7 @@ from xml.sax.handler import (feature_namespaces, feature_external_ges, LexicalHandler) from xml.sax.xmlreader import InputSource, AttributesImpl, AttributesNSImpl +from xml import sax from io import BytesIO, StringIO import codecs import os.path @@ -25,7 +26,7 @@ from urllib.error import URLError import urllib.request from test.support import os_helper -from test.support import findfile +from test.support import findfile, check__all__ from test.support.os_helper import FakePath, TESTFN @@ -1557,5 +1558,20 @@ def characters(self, content): self.assertEqual(self.char_index, 2) +class TestModuleAll(unittest.TestCase): + def test_all(self): + extra = ( + 'ContentHandler', + 'ErrorHandler', + 'InputSource', + 'SAXException', + 'SAXNotRecognizedException', + 'SAXNotSupportedException', + 'SAXParseException', + 'SAXReaderNotAvailable', + ) + check__all__(self, sax, extra=extra) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/xml/sax/__init__.py b/Lib/xml/sax/__init__.py index b657310207cfe5..fe4582c6f8b758 100644 --- a/Lib/xml/sax/__init__.py +++ b/Lib/xml/sax/__init__.py @@ -21,9 +21,9 @@ from .xmlreader import InputSource from .handler import ContentHandler, ErrorHandler -from ._exceptions import SAXException, SAXNotRecognizedException, \ - SAXParseException, SAXNotSupportedException, \ - SAXReaderNotAvailable +from ._exceptions import (SAXException, SAXNotRecognizedException, + SAXParseException, SAXNotSupportedException, + SAXReaderNotAvailable) def parse(source, handler, errorHandler=ErrorHandler()): @@ -55,7 +55,7 @@ def parseString(string, handler, errorHandler=ErrorHandler()): # tell modulefinder that importing sax potentially imports expatreader _false = 0 if _false: - import xml.sax.expatreader + import xml.sax.expatreader # noqa: F401 import os, sys if not sys.flags.ignore_environment and "PY_SAX_PARSER" in os.environ: @@ -92,3 +92,9 @@ def make_parser(parser_list=()): def _create_parser(parser_name): drv_module = __import__(parser_name,{},{},['create_parser']) return drv_module.create_parser() + + +__all__ = ['ContentHandler', 'ErrorHandler', 'InputSource', 'SAXException', + 'SAXNotRecognizedException', 'SAXNotSupportedException', + 'SAXParseException', 'SAXReaderNotAvailable', + 'default_parser_list', 'make_parser', 'parse', 'parseString'] From b2e71ff4f8fa5b7d8117dd8125137aee3d01f015 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 14 Jun 2024 15:29:09 -0400 Subject: [PATCH 487/903] gh-120161: Fix a Crash in the _datetime Module (gh-120182) In gh-120009 I used an atexit hook to finalize the _datetime module's static types at interpreter shutdown. However, atexit hooks are executed very early in finalization, which is a problem in the few cases where a subclass of one of those static types is still alive until the final GC collection. The static builtin types don't have this probably because they are finalized toward the end, after the final GC collection. To avoid the problem for _datetime, I have applied a similar approach here. Also, credit goes to @mgorny and @neonene for the new tests. FYI, I would have liked to take a slightly cleaner approach with managed static types, but wanted to get a smaller fix in first for the sake of backporting. I'll circle back to the cleaner approach with a future change on the main branch. --- Include/internal/pycore_typeobject.h | 24 ++++-- Lib/test/datetimetester.py | 44 +++++++++- ...-06-06-17-24-43.gh-issue-120161.DahNXV.rst | 2 + Modules/_datetimemodule.c | 48 +---------- Objects/typeobject.c | 85 +++++++++++++++---- Python/pylifecycle.c | 1 + 6 files changed, 133 insertions(+), 71 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index bc295b1b066bd1..32bd19d968b917 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -17,11 +17,25 @@ extern "C" { #define _Py_TYPE_BASE_VERSION_TAG (2<<16) #define _Py_MAX_GLOBAL_TYPE_VERSION_TAG (_Py_TYPE_BASE_VERSION_TAG - 1) +/* For now we hard-code this to a value for which we are confident + all the static builtin types will fit (for all builds). */ +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 +#define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 +#define _Py_MAX_MANAGED_STATIC_TYPES \ + (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) + struct _types_runtime_state { /* Used to set PyTypeObject.tp_version_tag for core static types. */ // bpo-42745: next_version_tag remains shared by all interpreters // because of static types. unsigned int next_version_tag; + + struct { + struct { + PyTypeObject *type; + int64_t interp_count; + } types[_Py_MAX_MANAGED_STATIC_TYPES]; + } managed_static; }; @@ -42,11 +56,6 @@ struct type_cache { struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; }; -/* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 -#define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 - typedef struct { PyTypeObject *type; int isbuiltin; @@ -133,6 +142,7 @@ struct types_state { extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); extern void _PyTypes_FiniTypes(PyInterpreterState *); +extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp); extern void _PyTypes_Fini(PyInterpreterState *); extern void _PyTypes_AfterFork(void); @@ -171,10 +181,6 @@ extern managed_static_type_state * _PyStaticType_GetState( PyAPI_FUNC(int) _PyStaticType_InitForExtension( PyInterpreterState *interp, PyTypeObject *self); -PyAPI_FUNC(void) _PyStaticType_FiniForExtension( - PyInterpreterState *interp, - PyTypeObject *self, - int final); /* Like PyType_GetModuleState, but skips verification diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 45188731eed688..70e2e2cccdc55f 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -23,7 +23,7 @@ from test import support from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST -from test.support import warnings_helper +from test.support import script_helper, warnings_helper import datetime as datetime_module from datetime import MINYEAR, MAXYEAR @@ -6822,6 +6822,48 @@ def run(type_checker, obj): self.assertEqual(ret, 0) +class ExtensionModuleTests(unittest.TestCase): + + def setUp(self): + if self.__class__.__name__.endswith('Pure'): + self.skipTest('Not relevant in pure Python') + + @support.cpython_only + def test_gh_120161(self): + with self.subTest('simple'): + script = textwrap.dedent(""" + import datetime + from _ast import Tuple + f = lambda: None + Tuple.dims = property(f, f) + + class tzutc(datetime.tzinfo): + pass + """) + script_helper.assert_python_ok('-c', script) + + with self.subTest('complex'): + script = textwrap.dedent(""" + import asyncio + import datetime + from typing import Type + + class tzutc(datetime.tzinfo): + pass + _EPOCHTZ = datetime.datetime(1970, 1, 1, tzinfo=tzutc()) + + class FakeDateMeta(type): + def __instancecheck__(self, obj): + return True + class FakeDate(datetime.date, metaclass=FakeDateMeta): + pass + def pickle_fake_date(datetime_) -> Type[FakeDate]: + # A pickle function for FakeDate + return FakeDate + """) + script_helper.assert_python_ok('-c', script) + + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) return standard_tests diff --git a/Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst b/Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst new file mode 100644 index 00000000000000..c378cac44c97bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst @@ -0,0 +1,2 @@ +:mod:`datetime` no longer crashes in certain complex reference cycle +situations. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index cb4622893375d7..5c4f1f888d17ee 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7129,37 +7129,6 @@ clear_state(datetime_state *st) } -/* --------------------------------------------------------------------------- - * Global module state. - */ - -// If we make _PyStaticType_*ForExtension() public -// then all this should be managed by the runtime. - -static struct { - PyMutex mutex; - int64_t interp_count; -} _globals = {0}; - -static void -callback_for_interp_exit(void *Py_UNUSED(data)) -{ - PyInterpreterState *interp = PyInterpreterState_Get(); - - assert(_globals.interp_count > 0); - PyMutex_Lock(&_globals.mutex); - _globals.interp_count -= 1; - int final = !_globals.interp_count; - PyMutex_Unlock(&_globals.mutex); - - /* They must be done in reverse order so subclasses are finalized - * before base classes. */ - for (size_t i = Py_ARRAY_LENGTH(capi_types); i > 0; i--) { - PyTypeObject *type = capi_types[i-1]; - _PyStaticType_FiniForExtension(interp, type, final); - } -} - static int init_static_types(PyInterpreterState *interp, int reloading) { @@ -7182,19 +7151,6 @@ init_static_types(PyInterpreterState *interp, int reloading) } } - PyMutex_Lock(&_globals.mutex); - assert(_globals.interp_count >= 0); - _globals.interp_count += 1; - PyMutex_Unlock(&_globals.mutex); - - /* It could make sense to add a separate callback - * for each of the types. However, for now we can take the simpler - * approach of a single callback. */ - if (PyUnstable_AtExit(interp, callback_for_interp_exit, NULL) < 0) { - callback_for_interp_exit(NULL); - return -1; - } - return 0; } @@ -7379,8 +7335,8 @@ module_clear(PyObject *mod) PyInterpreterState *interp = PyInterpreterState_Get(); clear_current_module(interp, mod); - // We take care of the static types via an interpreter atexit hook. - // See callback_for_interp_exit() above. + // The runtime takes care of the static types for us. + // See _PyTypes_FiniExtTypes().. return 0; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 8ecab555454cdc..98e00bd25c3205 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -159,18 +159,28 @@ managed_static_type_index_clear(PyTypeObject *self) self->tp_subclasses = NULL; } -static inline managed_static_type_state * -static_builtin_state_get(PyInterpreterState *interp, PyTypeObject *self) +static PyTypeObject * +static_ext_type_lookup(PyInterpreterState *interp, size_t index, + int64_t *p_interp_count) { - return &(interp->types.builtins.initialized[ - managed_static_type_index_get(self)]); -} + assert(interp->runtime == &_PyRuntime); + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); -static inline managed_static_type_state * -static_ext_type_state_get(PyInterpreterState *interp, PyTypeObject *self) -{ - return &(interp->types.for_extensions.initialized[ - managed_static_type_index_get(self)]); + size_t full_index = index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + int64_t interp_count = + _PyRuntime.types.managed_static.types[full_index].interp_count; + assert((interp_count == 0) == + (_PyRuntime.types.managed_static.types[full_index].type == NULL)); + *p_interp_count = interp_count; + + PyTypeObject *type = interp->types.for_extensions.initialized[index].type; + if (type == NULL) { + return NULL; + } + assert(!interp->types.for_extensions.initialized[index].isbuiltin); + assert(type == _PyRuntime.types.managed_static.types[full_index].type); + assert(managed_static_type_index_is_set(type)); + return type; } static managed_static_type_state * @@ -202,6 +212,8 @@ static void managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, int isbuiltin, int initial) { + assert(interp->runtime == &_PyRuntime); + size_t index; if (initial) { assert(!managed_static_type_index_is_set(self)); @@ -228,6 +240,21 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); } } + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + + assert((initial == 1) == + (_PyRuntime.types.managed_static.types[full_index].interp_count == 0)); + _PyRuntime.types.managed_static.types[full_index].interp_count += 1; + + if (initial) { + assert(_PyRuntime.types.managed_static.types[full_index].type == NULL); + _PyRuntime.types.managed_static.types[full_index].type = self; + } + else { + assert(_PyRuntime.types.managed_static.types[full_index].type == self); + } managed_static_type_state *state = isbuiltin ? &(interp->types.builtins.initialized[index]) @@ -256,15 +283,28 @@ static void managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, int isbuiltin, int final) { + size_t index = managed_static_type_index_get(self); + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + managed_static_type_state *state = isbuiltin - ? static_builtin_state_get(interp, self) - : static_ext_type_state_get(interp, self); + ? &(interp->types.builtins.initialized[index]) + : &(interp->types.for_extensions.initialized[index]); + assert(state != NULL); + + assert(_PyRuntime.types.managed_static.types[full_index].interp_count > 0); + assert(_PyRuntime.types.managed_static.types[full_index].type == state->type); assert(state->type != NULL); state->type = NULL; assert(state->tp_weaklist == NULL); // It was already cleared out. + _PyRuntime.types.managed_static.types[full_index].interp_count -= 1; if (final) { + assert(!_PyRuntime.types.managed_static.types[full_index].interp_count); + _PyRuntime.types.managed_static.types[full_index].type = NULL; + managed_static_type_index_clear(self); } @@ -840,8 +880,12 @@ _PyTypes_Fini(PyInterpreterState *interp) struct type_cache *cache = &interp->types.type_cache; type_cache_clear(cache, NULL); + // All the managed static types should have been finalized already. + assert(interp->types.for_extensions.num_initialized == 0); + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + assert(interp->types.for_extensions.initialized[i].type == NULL); + } assert(interp->types.builtins.num_initialized == 0); - // All the static builtin types should have been finalized already. for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { assert(interp->types.builtins.initialized[i].type == NULL); } @@ -5834,9 +5878,20 @@ fini_static_type(PyInterpreterState *interp, PyTypeObject *type, } void -_PyStaticType_FiniForExtension(PyInterpreterState *interp, PyTypeObject *type, int final) +_PyTypes_FiniExtTypes(PyInterpreterState *interp) { - fini_static_type(interp, type, 0, final); + for (size_t i = _Py_MAX_MANAGED_STATIC_EXT_TYPES; i > 0; i--) { + if (interp->types.for_extensions.num_initialized == 0) { + break; + } + int64_t count = 0; + PyTypeObject *type = static_ext_type_lookup(interp, i-1, &count); + if (type == NULL) { + continue; + } + int final = (count == 1); + fini_static_type(interp, type, 0, final); + } } void diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index cbdf5c1b771fff..3639cf6712053e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1818,6 +1818,7 @@ flush_std_files(void) static void finalize_interp_types(PyInterpreterState *interp) { + _PyTypes_FiniExtTypes(interp); _PyUnicode_FiniTypes(interp); _PySys_FiniTypes(interp); _PyXI_FiniTypes(interp); From e3b6cff33122554de0ef598664f5cd98de4fed6b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 14 Jun 2024 18:12:35 -0400 Subject: [PATCH 488/903] gh-120524: Temporarily Skip test_create_many_threaded In test_interpreters.test_stress (gh-120525) --- Lib/test/test_interpreters/test_stress.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index e400535b2a0e4e..40d2d77a7b9d3e 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -22,6 +22,7 @@ def test_create_many_sequential(self): interp = interpreters.create() alive.append(interp) + @unittest.skip('(temporary) gh-120524: there is a race that needs fixing') @support.requires_resource('cpu') def test_create_many_threaded(self): alive = [] From 92f6d400f76b6a04dddd944568870f689c8fab5f Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Sat, 15 Jun 2024 08:05:18 +0800 Subject: [PATCH 489/903] gh-119819: Conditional skip of logging tests that require multiprocessing subprocess support (#120476) Skip tests that require multiprocessing subprocess support. --- Lib/test/test_logging.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index ef2d4a621be962..504862ad53395e 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3898,6 +3898,7 @@ def do_queuehandler_configuration(self, qspec, lspec): self.addCleanup(os.remove, fn) @threading_helper.requires_working_threading() + @support.requires_subprocess() def test_config_queue_handler(self): q = CustomQueue() dq = { @@ -3926,12 +3927,10 @@ def test_config_queue_handler(self): msg = str(ctx.exception) self.assertEqual(msg, "Unable to configure handler 'ah'") + @support.requires_subprocess() def test_multiprocessing_queues(self): # See gh-119819 - # will skip test if it's not available - import_helper.import_module('_multiprocessing') - cd = copy.deepcopy(self.config_queue_handler) from multiprocessing import Queue as MQ, Manager as MM q1 = MQ() # this can't be pickled From 5c58e728b1391c258b224fc6d88f62f42c725026 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Sat, 15 Jun 2024 08:05:30 +0800 Subject: [PATCH 490/903] gh-117398: Use the correct module loader for iOS in datetime CAPI test. (#120477) Use the correct loader for iOS. --- Lib/test/datetimetester.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 70e2e2cccdc55f..e55b738eb4a975 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -6786,6 +6786,13 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) def test_type_check_in_subinterp(self): + # iOS requires the use of the custom framework loader, + # not the ExtensionFileLoader. + if sys.platform == "ios": + extension_loader = "AppleFrameworkLoader" + else: + extension_loader = "ExtensionFileLoader" + script = textwrap.dedent(f""" if {_interpreters is None}: import _testcapi as module @@ -6795,7 +6802,7 @@ def test_type_check_in_subinterp(self): import importlib.util fullname = '_testcapi_datetime' origin = importlib.util.find_spec('_testcapi').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + loader = importlib.machinery.{extension_loader}(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) From d4039d3f6f8cb7738c5cd272dde04171446dfd2b Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Fri, 14 Jun 2024 22:33:09 -0700 Subject: [PATCH 491/903] gh-120526: Correct signature of map() builtin (GH-120528) map() requires at least one iterable arg. Signed-off-by: Adam Williamson --- Python/bltinmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c4d3ecbeeff0e6..6e50623cafa4ed 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1475,7 +1475,7 @@ static PyMethodDef map_methods[] = { PyDoc_STRVAR(map_doc, -"map(function, /, *iterables)\n\ +"map(function, iterable, /, *iterables)\n\ --\n\ \n\ Make an iterator that computes the function using arguments from\n\ From 42ebdd83bb194f054fe5a10b3caa0c3a95be3679 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 15 Jun 2024 13:33:14 +0300 Subject: [PATCH 492/903] gh-120544: Add `else: fail()` to tests where exception is expected (#120545) --- Lib/test/test_exceptions.py | 2 ++ Lib/test/test_unittest/test_case.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 9460d1f1c864b9..e4f2e3a97b8bb8 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1859,6 +1859,8 @@ def f(): except self.failureException: with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) + else: + self.fail("assertRaisesRegex should have failed.") self.assertIn("aab", err.getvalue()) diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index 17420909402107..b4b2194a09cf9f 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1151,6 +1151,8 @@ def testAssertMultiLineEqual(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualSingleLine(self): sample_text = "laden swallows fly slowly" @@ -1167,6 +1169,8 @@ def testAssertEqualSingleLine(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualwithEmptyString(self): '''Verify when there is an empty string involved, the diff output @@ -1184,6 +1188,8 @@ def testAssertEqualwithEmptyString(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualMultipleLinesMissingNewlineTerminator(self): '''Verifying format of diff output from assertEqual involving strings @@ -1204,6 +1210,8 @@ def testAssertEqualMultipleLinesMissingNewlineTerminator(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): '''Verifying format of diff output from assertEqual involving strings @@ -1227,6 +1235,8 @@ def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testEqualityBytesWarning(self): if sys.flags.bytes_warning: From c501261c919ceb97c850ef9427a93326f06a8f2e Mon Sep 17 00:00:00 2001 From: Wulian233 <71213467+Wulian233@users.noreply.github.com> Date: Sat, 15 Jun 2024 19:04:14 +0800 Subject: [PATCH 493/903] gh-120495: Fix incorrect exception handling in Tab Nanny (#120498) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/tabnanny.py | 8 ++++---- Lib/test/test_tabnanny.py | 2 +- Misc/ACKS | 1 + .../2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 7e56d4a48d1d00..c0097351b269f2 100644 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -105,14 +105,14 @@ def check(file): errprint("%r: Token Error: %s" % (file, msg)) return - except SyntaxError as msg: - errprint("%r: Token Error: %s" % (file, msg)) - return - except IndentationError as msg: errprint("%r: Indentation Error: %s" % (file, msg)) return + except SyntaxError as msg: + errprint("%r: Syntax Error: %s" % (file, msg)) + return + except NannyNag as nag: badline = nag.get_lineno() line = nag.get_line() diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index cc122cafc7985c..30dcb3e3c4f4f9 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -315,7 +315,7 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failur def test_with_errored_file(self): """Should displays error when errored python file is given.""" with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: - stderr = f"{file_path!r}: Token Error: " + stderr = f"{file_path!r}: Indentation Error: " stderr += ('unindent does not match any outer indentation level' ' (, line 3)') self.validate_cmd(file_path, stderr=stderr, expect_failure=True) diff --git a/Misc/ACKS b/Misc/ACKS index 2f4c0793437fb6..a406fca8744a5f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1099,6 +1099,7 @@ Ivan Levkivskyi Ben Lewis William Lewis Akira Li +Jiahao Li Robert Li Xuanji Li Zekun Li diff --git a/Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst b/Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst new file mode 100644 index 00000000000000..d5114c3d3c904c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst @@ -0,0 +1 @@ +Fix incorrect exception handling in Tab Nanny. Patch by Wulian233. From 99d62f902e43c08ebec5a292fd3b30a9fc4cba69 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 15 Jun 2024 13:51:58 +0100 Subject: [PATCH 494/903] Add some more edge-case tests for `inspect.get_annotations` with `eval_str=True` (#120550) --- .../inspect_stringized_annotations_pep695.py | 23 ++++++++++++++---- Lib/test/test_inspect/test_inspect.py | 24 +++++++++++++------ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py index 723822f8eaa92d..39bfe2edb03f30 100644 --- a/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py +++ b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py @@ -45,6 +45,13 @@ def generic_method[Foo, **Bar]( def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass +# Eggs is `int` in globals, a TypeVar in type_params, and `str` in locals: +class E[Eggs]: + Eggs = str + x: Eggs + + + def nested(): from types import SimpleNamespace from inspect import get_annotations @@ -53,7 +60,7 @@ def nested(): Spam = memoryview - class E[Eggs, **Spam]: + class F[Eggs, **Spam]: x: Eggs y: Spam @@ -63,10 +70,18 @@ def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + # Eggs is `int` in globals, `bytes` in the function scope, + # a TypeVar in the type_params, and `str` in locals: + class G[Eggs]: + Eggs = str + x: Eggs + + return SimpleNamespace( - E=E, - E_annotations=get_annotations(E, eval_str=True), - E_meth_annotations=get_annotations(E.generic_method, eval_str=True), + F=F, + F_annotations=get_annotations(F, eval_str=True), + F_meth_annotations=get_annotations(F.generic_method, eval_str=True), + G_annotations=get_annotations(G, eval_str=True), generic_func=generic_function, generic_func_annotations=get_annotations(generic_function, eval_str=True) ) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 140efac530afb2..ea8735d8f06459 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1770,26 +1770,36 @@ def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_v ) ) + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars(self): + self.assertEqual( + inspect.get_annotations( + inspect_stringized_annotations_pep695.E, eval_str=True + ), + {"x": str}, + ) + def test_pep_695_generics_with_future_annotations_nested_in_function(self): results = inspect_stringized_annotations_pep695.nested() self.assertEqual( - set(results.E_annotations.values()), - set(results.E.__type_params__) + set(results.F_annotations.values()), + set(results.F.__type_params__) ) self.assertEqual( - set(results.E_meth_annotations.values()), - set(results.E.generic_method.__type_params__) + set(results.F_meth_annotations.values()), + set(results.F.generic_method.__type_params__) ) self.assertNotEqual( - set(results.E_meth_annotations.values()), - set(results.E.__type_params__) + set(results.F_meth_annotations.values()), + set(results.F.__type_params__) ) self.assertEqual( - set(results.E_meth_annotations.values()).intersection(results.E.__type_params__), + set(results.F_meth_annotations.values()).intersection(results.F.__type_params__), set() ) + self.assertEqual(results.G_annotations, {"x": str}) + self.assertEqual( set(results.generic_func_annotations.values()), set(results.generic_func.__type_params__) From 6f63dfff6f493b405f3422210a168369e1e7a35d Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sat, 15 Jun 2024 22:39:22 +0800 Subject: [PATCH 495/903] gh-117657: Make PyType_HasFeature (exported version) atomic (#120484) Make PyType_HasFeature (exported version) atomic --- Include/object.h | 6 +++++- Objects/typeobject.c | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Include/object.h b/Include/object.h index 4a39ada8c7daa4..f71aaee7efe6ee 100644 --- a/Include/object.h +++ b/Include/object.h @@ -756,7 +756,11 @@ PyType_HasFeature(PyTypeObject *type, unsigned long feature) // PyTypeObject is opaque in the limited C API flags = PyType_GetFlags(type); #else - flags = type->tp_flags; +# ifdef Py_GIL_DISABLED + flags = _Py_atomic_load_ulong_relaxed(&type->tp_flags); +# else + flags = type->tp_flags; +# endif #endif return ((flags & feature) != 0); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 98e00bd25c3205..eb296414bb7bef 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3599,7 +3599,7 @@ type_init(PyObject *cls, PyObject *args, PyObject *kwds) unsigned long PyType_GetFlags(PyTypeObject *type) { - return type->tp_flags; + return FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags); } From 9e0b11eb21930b7b8e4a396200a921e9985cfca4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 15 Jun 2024 08:18:16 -0700 Subject: [PATCH 496/903] annotations: expand documentation on "simple" assignment targets (#120535) This behavior is rather surprising and it was not clearly specified. Co-authored-by: Alex Waygood --- Doc/library/ast.rst | 10 +++++++--- Doc/reference/simple_stmts.rst | 7 +++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 9ee56b92431b57..f7e8afa7000392 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -891,9 +891,13 @@ Statements An assignment with a type annotation. ``target`` is a single node and can be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` - node. ``value`` is a single optional node. ``simple`` is a boolean integer - set to True for a :class:`Name` node in ``target`` that do not appear in - between parenthesis and are hence pure names and not expressions. + node. ``value`` is a single optional node. + + ``simple`` is always either 0 (indicating a "complex" target) or 1 + (indicating a "simple" target). A "simple" target consists solely of a + :class:`Name` node that does not appear between parentheses; all other + targets are considered complex. Only simple targets appear in + the :attr:`__annotations__` dictionary of modules and classes. .. doctest:: diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index a253482156d3b4..4f6c0c63ae42be 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -333,7 +333,9 @@ statement, of a variable or attribute annotation and an optional assignment stat The difference from normal :ref:`assignment` is that only a single target is allowed. -For simple names as assignment targets, if in class or module scope, +The assignment target is considered "simple" if it consists of a single +name that is not enclosed in parentheses. +For simple assignment targets, if in class or module scope, the annotations are evaluated and stored in a special class or module attribute :attr:`__annotations__` that is a dictionary mapping from variable names (mangled if private) to @@ -341,7 +343,8 @@ evaluated annotations. This attribute is writable and is automatically created at the start of class or module body execution, if annotations are found statically. -For expressions as assignment targets, the annotations are evaluated if +If the assignment target is not simple (an attribute, subscript node, or +parenthesized name), the annotation is evaluated if in class or module scope, but not stored. If a name is annotated in a function scope, then this name is local for From 31d1d72d7e24e0427df70f7dd14b9baff28a4f89 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 15 Jun 2024 20:56:40 +0300 Subject: [PATCH 497/903] gh-120541: Improve the "less" prompt in pydoc (GH-120543) When help() is called with non-string argument, use __qualname__ or __name__ if available, otherwise use "{typename} object". --- Lib/pydoc.py | 9 ++- Lib/test/test_pydoc/test_pydoc.py | 62 +++++++++++++++---- ...-06-15-12-04-46.gh-issue-120541.d3cc5y.rst | 2 + 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index be5cd9a80db710..768c3dcb11ec59 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1755,7 +1755,14 @@ def doc(thing, title='Python Library Documentation: %s', forceload=0, """Display text documentation, given an object or a path to an object.""" if output is None: try: - what = thing if isinstance(thing, str) else type(thing).__name__ + if isinstance(thing, str): + what = thing + else: + what = getattr(thing, '__qualname__', None) + if not isinstance(what, str): + what = getattr(thing, '__name__', None) + if not isinstance(what, str): + what = type(thing).__name__ + ' object' pager(render_doc(thing, title, forceload), f'Help on {what!s}') except ImportError as exc: if is_cli: diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index a17c16cc73cf0e..b520cfd0b50e38 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -31,7 +31,7 @@ from test.support.script_helper import (assert_python_ok, assert_python_failure, spawn_python) from test.support import threading_helper -from test.support import (reap_children, captured_output, captured_stdout, +from test.support import (reap_children, captured_stdout, captured_stderr, is_emscripten, is_wasi, requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) @@ -680,9 +680,8 @@ def test_help_output_redirect(self, pager_mock): help_header = textwrap.dedent(help_header) expected_help_pattern = help_header + expected_text_pattern - with captured_output('stdout') as output, \ - captured_output('stderr') as err, \ - StringIO() as buf: + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() helper = pydoc.Helper(output=buf) helper.help(module) result = buf.getvalue().strip() @@ -706,9 +705,8 @@ def test_help_output_redirect_various_requests(self, pager_mock): def run_pydoc_for_request(request, expected_text_part): """Helper function to run pydoc with its output redirected""" - with captured_output('stdout') as output, \ - captured_output('stderr') as err, \ - StringIO() as buf: + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() helper = pydoc.Helper(output=buf) helper.help(request) result = buf.getvalue().strip() @@ -742,6 +740,45 @@ def run_pydoc_for_request(request, expected_text_part): run_pydoc_for_request(pydoc.Helper.help, 'Help on function help in module pydoc:') # test for pydoc.Helper() instance skipped because it is always meant to be interactive + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_help_output_pager(self): + def run_pydoc_pager(request, what, expected_first_line): + with (captured_stdout() as output, + captured_stderr() as err, + unittest.mock.patch('pydoc.pager') as pager_mock, + self.subTest(repr(request))): + helper = pydoc.Helper() + helper.help(request) + self.assertEqual('', err.getvalue()) + self.assertEqual('\n', output.getvalue()) + pager_mock.assert_called_once() + result = clean_text(pager_mock.call_args.args[0]) + self.assertEqual(result.splitlines()[0], expected_first_line) + self.assertEqual(pager_mock.call_args.args[1], f'Help on {what}') + + run_pydoc_pager('%', 'EXPRESSIONS', 'Operator precedence') + run_pydoc_pager('True', 'bool object', 'Help on bool object:') + run_pydoc_pager(True, 'bool object', 'Help on bool object:') + run_pydoc_pager('assert', 'assert', 'The "assert" statement') + run_pydoc_pager('TYPES', 'TYPES', 'The standard type hierarchy') + run_pydoc_pager('pydoc.Helper.help', 'pydoc.Helper.help', + 'Help on function help in pydoc.Helper:') + run_pydoc_pager(pydoc.Helper.help, 'Helper.help', + 'Help on function help in module pydoc:') + run_pydoc_pager('str', 'str', 'Help on class str in module builtins:') + run_pydoc_pager(str, 'str', 'Help on class str in module builtins:') + run_pydoc_pager('str.upper', 'str.upper', 'Help on method_descriptor in str:') + run_pydoc_pager(str.upper, 'str.upper', 'Help on method_descriptor:') + run_pydoc_pager(str.__add__, 'str.__add__', 'Help on wrapper_descriptor:') + run_pydoc_pager(int.numerator, 'int.numerator', + 'Help on getset descriptor builtins.int.numerator:') + run_pydoc_pager(list[int], 'list', + 'Help on GenericAlias in module builtins:') + run_pydoc_pager('sys', 'sys', 'Help on built-in module sys:') + run_pydoc_pager(sys, 'sys', 'Help on built-in module sys:') + def test_showtopic(self): with captured_stdout() as showtopic_io: helper = pydoc.Helper() @@ -775,9 +812,8 @@ def test_showtopic_output_redirect(self, pager_mock): # Helper.showtopic should be redirected self.maxDiff = None - with captured_output('stdout') as output, \ - captured_output('stderr') as err, \ - StringIO() as buf: + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() helper = pydoc.Helper(output=buf) helper.showtopic('with') result = buf.getvalue().strip() @@ -790,7 +826,7 @@ def test_showtopic_output_redirect(self, pager_mock): def test_lambda_with_return_annotation(self): func = lambda a, b, c: 1 func.__annotations__ = {"return": int} - with captured_output('stdout') as help_io: + with captured_stdout() as help_io: pydoc.help(func) helptext = help_io.getvalue() self.assertIn("lambda (a, b, c) -> int", helptext) @@ -798,7 +834,7 @@ def test_lambda_with_return_annotation(self): def test_lambda_without_return_annotation(self): func = lambda a, b, c: 1 func.__annotations__ = {"a": int, "b": int, "c": int} - with captured_output('stdout') as help_io: + with captured_stdout() as help_io: pydoc.help(func) helptext = help_io.getvalue() self.assertIn("lambda (a: int, b: int, c: int)", helptext) @@ -806,7 +842,7 @@ def test_lambda_without_return_annotation(self): def test_lambda_with_return_and_params_annotation(self): func = lambda a, b, c: 1 func.__annotations__ = {"a": int, "b": int, "c": int, "return": int} - with captured_output('stdout') as help_io: + with captured_stdout() as help_io: pydoc.help(func) helptext = help_io.getvalue() self.assertIn("lambda (a: int, b: int, c: int) -> int", helptext) diff --git a/Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst b/Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst new file mode 100644 index 00000000000000..bf8830c6c50386 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst @@ -0,0 +1,2 @@ +Improve the prompt in the "less" pager when :func:`help` is called with +non-string argument. From 08d09cf5ba041c9c5c3860200b56bab66fd44a23 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Sat, 15 Jun 2024 20:46:39 +0200 Subject: [PATCH 498/903] gh-112346: Always set OS byte to 255, simpler gzip.compress function. (GH-120486) This matches the output behavior in 3.10 and earlier; the optimization in 3.11 allowed the zlib library's "os" value to be filled in instead in the circumstance when mtime was 0. this keeps things consistent. --- Doc/library/gzip.rst | 8 ++-- Lib/gzip.py | 38 ++++--------------- Lib/test/test_gzip.py | 12 +++++- ...4-06-12-10-00-31.gh-issue-90425.5CfkKG.rst | 2 + 4 files changed, 26 insertions(+), 34 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-12-10-00-31.gh-issue-90425.5CfkKG.rst diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 965da5981f6dbc..152cba4f653cb4 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -188,9 +188,7 @@ The module defines the following items: Compress the *data*, returning a :class:`bytes` object containing the compressed data. *compresslevel* and *mtime* have the same meaning as in - the :class:`GzipFile` constructor above. When *mtime* is set to ``0``, this - function is equivalent to :func:`zlib.compress` with *wbits* set to ``31``. - The zlib function is faster. + the :class:`GzipFile` constructor above. .. versionadded:: 3.2 .. versionchanged:: 3.8 @@ -200,6 +198,10 @@ The module defines the following items: streamed fashion. Calls with *mtime* set to ``0`` are delegated to :func:`zlib.compress` for better speed. + .. versionchanged:: 3.13 + The gzip header OS byte is guaranteed to be set to 255 when this function + is used as was the case in 3.10 and earlier. + .. function:: decompress(data) Decompress the *data*, returning a :class:`bytes` object containing the diff --git a/Lib/gzip.py b/Lib/gzip.py index 0d19c84c59cfa7..ba753ce3050dd8 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -580,27 +580,6 @@ def _rewind(self): self._new_member = True -def _create_simple_gzip_header(compresslevel: int, - mtime = None) -> bytes: - """ - Write a simple gzip header with no extra fields. - :param compresslevel: Compresslevel used to determine the xfl bytes. - :param mtime: The mtime (must support conversion to a 32-bit integer). - :return: A bytes object representing the gzip header. - """ - if mtime is None: - mtime = time.time() - if compresslevel == _COMPRESS_LEVEL_BEST: - xfl = 2 - elif compresslevel == _COMPRESS_LEVEL_FAST: - xfl = 4 - else: - xfl = 0 - # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra - # fields added to header), mtime, xfl and os (255 for unknown OS). - return struct.pack(" Date: Sun, 16 Jun 2024 13:36:10 +0800 Subject: [PATCH 499/903] gh-120572: add missing parentheses in TypeIs documentation (#120573) --- Doc/library/typing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 94de64fcf835fc..bf0ff9bd348553 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1454,8 +1454,8 @@ These can be used as types in annotations. They all support subscription using to write such functions in a type-safe manner. If a ``TypeIs`` function is a class or instance method, then the type in - ``TypeIs`` maps to the type of the second parameter after ``cls`` or - ``self``. + ``TypeIs`` maps to the type of the second parameter (after ``cls`` or + ``self``). In short, the form ``def foo(arg: TypeA) -> TypeIs[TypeB]: ...``, means that if ``foo(arg)`` returns ``True``, then ``arg`` is an instance From cf49ef78f894e418bea7de23dde9b01d6235889d Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 16 Jun 2024 01:55:47 -0400 Subject: [PATCH 500/903] gh-120360: Add self as IDLE doc owner (#120571) Add self as IDLE doc owner --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1f9047ab97e934..eb7cc88565f6d0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -212,6 +212,7 @@ Doc/c-api/stable.rst @encukou **/*ensurepip* @pfmoore @pradyunsg **/*idlelib* @terryjreedy +/Doc/library/idle.rst @terryjreedy **/*typing* @JelleZijlstra @AlexWaygood From 0c0348adbfca991f78b3aaa6790e5c26606a1c0f Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 16 Jun 2024 11:26:13 +0300 Subject: [PATCH 501/903] gh-120579: Guard `_testcapi` import in `test_free_threading` (#120580) --- Lib/test/test_free_threading/test_dict.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index f877582e6b565c..3126458e08e50a 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -8,7 +8,10 @@ from threading import Thread from unittest import TestCase -from _testcapi import dict_version +try: + import _testcapi +except ImportError: + _testcapi = None from test.support import threading_helper @@ -139,7 +142,9 @@ def writer_func(l): for ref in thread_list: self.assertIsNone(ref()) + @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_dict_version(self): + dict_version = _testcapi.dict_version THREAD_COUNT = 10 DICT_COUNT = 10000 lists = [] From 192d17c3fd9945104bc0303cf248bb0d074d260e Mon Sep 17 00:00:00 2001 From: Idan Kapustian <71190257+idankap@users.noreply.github.com> Date: Sun, 16 Jun 2024 15:15:03 +0300 Subject: [PATCH 502/903] gh-120485: Add an override of `allow_reuse_port` on classes subclassing `socketserver.TCPServer` (GH-120488) Co-authored-by: Vinay Sajip --- Lib/http/server.py | 3 ++- Lib/logging/config.py | 3 ++- Lib/test/test_logging.py | 1 + Lib/xmlrpc/server.py | 1 + .../2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst | 1 + 5 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst diff --git a/Lib/http/server.py b/Lib/http/server.py index 7d0da5052d2d4d..2d010649e56b51 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -129,7 +129,8 @@ class HTTPServer(socketserver.TCPServer): - allow_reuse_address = 1 # Seems to make sense in testing environment + allow_reuse_address = True # Seems to make sense in testing environment + allow_reuse_port = True def server_bind(self): """Override server_bind to store the server name.""" diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 9de84e527b18ac..d2f23e53f35c57 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -984,7 +984,8 @@ class ConfigSocketReceiver(ThreadingTCPServer): A simple TCP socket-based logging config receiver. """ - allow_reuse_address = 1 + allow_reuse_address = True + allow_reuse_port = True def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 504862ad53395e..5192ce252a4d4c 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1038,6 +1038,7 @@ class TestTCPServer(ControlMixin, ThreadingTCPServer): """ allow_reuse_address = True + allow_reuse_port = True def __init__(self, addr, handler, poll_interval=0.5, bind_and_activate=True): diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 4dddb1d10e08bd..90a356fbb8eae4 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -578,6 +578,7 @@ class SimpleXMLRPCServer(socketserver.TCPServer, """ allow_reuse_address = True + allow_reuse_port = True # Warning: this is for debugging purposes only! Never set this to True in # production code, as will be sending out sensitive information (exception diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst new file mode 100644 index 00000000000000..f41c233908362f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst @@ -0,0 +1 @@ +Add an override of ``allow_reuse_port`` on classes subclassing ``socketserver.TCPServer`` where ``allow_reuse_address`` is also overridden. From b8484c6ad7fd14ca464e584b79821b4b906dd77a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 16 Jun 2024 06:51:17 -0600 Subject: [PATCH 503/903] Docs: remove temporary hardcoded links (#120348) --- Doc/tools/static/rtd_switcher.js | 35 +------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/Doc/tools/static/rtd_switcher.js b/Doc/tools/static/rtd_switcher.js index a67bb85505a9ca..f5dc7045a0dbc4 100644 --- a/Doc/tools/static/rtd_switcher.js +++ b/Doc/tools/static/rtd_switcher.js @@ -6,42 +6,9 @@ document.addEventListener("readthedocs-addons-data-ready", function(event) { const config = event.detail.data() - - // Add some mocked hardcoded versions pointing to the official - // documentation while migrating to Read the Docs. - // These are only for testing purposes. - // TODO: remove them when managing all the versions on Read the Docs, - // since all the "active, built and not hidden" versions will be shown automatically. - let versions = config.versions.active.concat([ - { - slug: "dev (3.14)", - urls: { - documentation: "https://docs.python.org/3.14/", - } - }, - { - slug: "dev (3.13)", - urls: { - documentation: "https://docs.python.org/3.13/", - } - }, - { - slug: "3.12", - urls: { - documentation: "https://docs.python.org/3.12/", - } - }, - { - slug: "3.11", - urls: { - documentation: "https://docs.python.org/3.11/", - } - }, - ]); - const versionSelect = `