From a028778d4c813914ae1e6ef3a04bb96dbf92ace6 Mon Sep 17 00:00:00 2001 From: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Date: Tue, 14 Mar 2023 22:16:45 +0000 Subject: [PATCH 01/34] Rename redundant enum tests so that they run (#102535) --- Lib/test/test_enum.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 2b14590b2c21af..a11bb441f06e8e 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4579,15 +4579,14 @@ class Double(Enum): TWO = 2 self.assertEqual(Double.__doc__, None) - - def test_doc_1(self): + def test_doc_3(self): class Triple(Enum): ONE = 1 TWO = 2 THREE = 3 self.assertEqual(Triple.__doc__, None) - def test_doc_1(self): + def test_doc_4(self): class Quadruple(Enum): ONE = 1 TWO = 2 From 152292b98fd52f8b3db252e5609d6a605a11cb57 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 14 Mar 2023 22:38:15 +0000 Subject: [PATCH 02/34] gh-101578: mention in what's new in 3.12 that exceptions are now normalized before stored (#102702) --- Doc/whatsnew/3.12.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 9f33dbc808ddc0..217ffec1ee1007 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -982,6 +982,11 @@ Porting to Python 3.12 effects, these side effects are no longer duplicated. (Contributed by Victor Stinner in :gh:`98724`.) +* The interpreter's error indicator is now always normalized. This means + that :c:func:`PyErr_SetObject`, :c:func:`PyErr_SetString` and the other + functions that set the error indicator now normalize the exception + before storing it. (Contributed by Mark Shannon in :gh:`101578`.) + Deprecated ---------- From 0a539b5db312d126ff45dd4aa6a53d40a292c512 Mon Sep 17 00:00:00 2001 From: "Robert Prater (B. Eng)" Date: Tue, 14 Mar 2023 20:03:43 -0400 Subject: [PATCH 03/34] gh-102703: Fix typo in modules tutorial documentation (GH-102707) **Before** This prevents directories with a common name, such as ``string``, unintentionally hiding ... **After** This prevents directories with a common name, such as ``string``, from unintentionally hiding ... --- Doc/tutorial/modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index ad70d92994af49..4daafa49a34d2e 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -438,7 +438,7 @@ When importing the package, Python searches through the directories on The :file:`__init__.py` files are required to make Python treat directories containing the file as packages. This prevents directories with a common name, -such as ``string``, unintentionally hiding valid modules that occur later +such as ``string``, from unintentionally hiding valid modules that occur later on the module search path. In the simplest case, :file:`__init__.py` can just be an empty file, but it can also execute initialization code for the package or set the ``__all__`` variable, described later. From 5fce813d8e547d6508daa386b67f230105c3a174 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 15 Mar 2023 00:07:30 +0000 Subject: [PATCH 04/34] gh-102519: Avoid failing tests due to inaccessible volumes (GH-102706) --- Lib/test/test_os.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 253e2a23238f12..42357fef80ec89 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2683,12 +2683,17 @@ def test_listvolumes(self): def test_listmounts(self): for volume in os.listvolumes(): - mounts = os.listmounts(volume) - self.assertIsInstance(mounts, list) - self.assertSetEqual( - set(mounts), - self.known_mounts & set(mounts), - ) + try: + mounts = os.listmounts(volume) + except OSError as ex: + if support.verbose: + print("Skipping", volume, "because of", ex) + else: + self.assertIsInstance(mounts, list) + self.assertSetEqual( + set(mounts), + self.known_mounts & set(mounts), + ) @unittest.skipUnless(hasattr(os, 'readlink'), 'needs os.readlink()') From e94edab727d07bef851e0e1a345e18a453be863d Mon Sep 17 00:00:00 2001 From: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Date: Wed, 15 Mar 2023 00:33:19 +0000 Subject: [PATCH 05/34] gh-102560 Add docstrings to asyncio.TaskGroup (#102565) --- Lib/asyncio/taskgroups.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 911419e1769c17..0fdea3697ece3d 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -10,7 +10,21 @@ class TaskGroup: + """Asynchronous context manager for managing groups of tasks. + Example use: + + async with asyncio.TaskGroup() as group: + task1 = group.create_task(some_coroutine(...)) + task2 = group.create_task(other_coroutine(...)) + print("Both tasks have completed now.") + + All tasks are awaited when the context manager exits. + + Any exceptions other than `asyncio.CancelledError` raised within + a task will cancel all remaining tasks and wait for them to exit. + The exceptions are then combined and raised as an `ExceptionGroup`. + """ def __init__(self): self._entered = False self._exiting = False @@ -135,6 +149,10 @@ async def __aexit__(self, et, exc, tb): self._errors = None def create_task(self, coro, *, name=None, context=None): + """Create a new task in this group and return it. + + Similar to `asyncio.create_task`. + """ if not self._entered: raise RuntimeError(f"TaskGroup {self!r} has not been entered") if self._exiting and not self._tasks: From 5e0865f22eed9f3f3f0e912c4ada196effbd8ce0 Mon Sep 17 00:00:00 2001 From: Andre Hora Date: Tue, 14 Mar 2023 23:36:31 -0300 Subject: [PATCH 06/34] gh-101377: improving test_locale_calendar_formatweekday of calendar (#101378) --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Terry Jan Reedy --- Lib/test/test_calendar.py | 8 ++++++-- .../Tests/2023-01-27-18-10-40.gh-issue-101377.IJGpqh.rst | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-01-27-18-10-40.gh-issue-101377.IJGpqh.rst diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 3d9dcf12f2dad8..ccfbeede0be949 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -568,11 +568,15 @@ def test_locale_calendar_formatweekday(self): try: # formatweekday uses different day names based on the available width. cal = calendar.LocaleTextCalendar(locale='en_US') + # For really short widths, the abbreviated name is truncated. + self.assertEqual(cal.formatweekday(0, 1), "M") + self.assertEqual(cal.formatweekday(0, 2), "Mo") # For short widths, a centered, abbreviated name is used. + self.assertEqual(cal.formatweekday(0, 3), "Mon") self.assertEqual(cal.formatweekday(0, 5), " Mon ") - # For really short widths, even the abbreviated name is truncated. - self.assertEqual(cal.formatweekday(0, 2), "Mo") + self.assertEqual(cal.formatweekday(0, 8), " Mon ") # For long widths, the full day name is used. + self.assertEqual(cal.formatweekday(0, 9), " Monday ") self.assertEqual(cal.formatweekday(0, 10), " Monday ") except locale.Error: raise unittest.SkipTest('cannot set the en_US locale') diff --git a/Misc/NEWS.d/next/Tests/2023-01-27-18-10-40.gh-issue-101377.IJGpqh.rst b/Misc/NEWS.d/next/Tests/2023-01-27-18-10-40.gh-issue-101377.IJGpqh.rst new file mode 100644 index 00000000000000..a9c19ce060e3ab --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-01-27-18-10-40.gh-issue-101377.IJGpqh.rst @@ -0,0 +1 @@ +Improved test_locale_calendar_formatweekday of calendar. From 8647ba4b639077e201751ae6dbd82e8bfcf80895 Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Wed, 15 Mar 2023 20:06:32 +1300 Subject: [PATCH 07/34] Remove misformatted exclamation marks in docs (#102694) Remove the exclamation mark from :program:`!foo` in .rst files because it inadvertently shows up in the rendered HTML. (Sphinx's cross-referencing roles use a '!' prefix to suppress hyperlinking[1], but :program: is not a cross-referencing role so the '!' is displayed verbatim.) The exclamation marks in venv.rst were introduced in #98350. See comments [2] and [3] for additional discussion. [1]: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#cross-referencing-syntax [2]: https://github.com/python/cpython/pull/98350#issuecomment-1285965759 [3]: https://github.com/python/cpython/pull/98350#issuecomment-1286394047 Reported-by: Vinay Sajip --- Doc/library/venv.rst | 4 ++-- Misc/NEWS.d/3.12.0a2.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 8eb0b35eaa12df..240ab139838db9 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -61,7 +61,7 @@ running from a virtual environment. A virtual environment may be "activated" using a script in its binary directory (``bin`` on POSIX; ``Scripts`` on Windows). This will prepend that directory to your :envvar:`!PATH`, so that running -:program:`!python` will invoke the environment's Python interpreter +:program:`python` will invoke the environment's Python interpreter and you can run installed scripts without having to use their full path. The invocation of the activation script is platform-specific (:samp:`{}` must be replaced by the path to the directory @@ -84,7 +84,7 @@ containing the virtual environment): +-------------+------------+--------------------------------------------------+ .. versionadded:: 3.4 - :program:`!fish` and :program:`!csh` activation scripts. + :program:`fish` and :program:`csh` activation scripts. .. versionadded:: 3.8 PowerShell activation scripts installed under POSIX for PowerShell Core diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 117be21a3221b6..d871384903e7cd 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -959,7 +959,7 @@ Fix ``make regen-test-levenshtein`` for out-of-tree builds. Don't use vendored ``libmpdec`` headers if :option:`--with-system-libmpdec` is passed to :program:`configure`. Don't use vendored ``libexpat`` headers -if :option:`--with-system-expat` is passed to :program:`!configure`. +if :option:`--with-system-expat` is passed to :program:`configure`. .. From 2b5781d659ce3ffe03d4c1f46e4875e604cf2d88 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 15 Mar 2023 12:33:41 +0300 Subject: [PATCH 08/34] gh-102615: Use `list` instead of `tuple` in `repr` of paramspec (#102637) Co-authored-by: Alex Waygood --- Lib/test/test_typing.py | 45 +++++++++++++++++++ Lib/typing.py | 7 +-- ...-03-13-12-05-55.gh-issue-102615.NcA_ZL.rst | 3 ++ 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-13-12-05-55.gh-issue-102615.NcA_ZL.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c17be6cd0bbc4a..89c725cda54f12 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3809,6 +3809,51 @@ class Y(C[int]): self.assertEqual(Y.__qualname__, 'GenericTests.test_repr_2..Y') + def test_repr_3(self): + T = TypeVar('T') + T1 = TypeVar('T1') + P = ParamSpec('P') + P2 = ParamSpec('P2') + Ts = TypeVarTuple('Ts') + + class MyCallable(Generic[P, T]): + pass + + class DoubleSpec(Generic[P, P2, T]): + pass + + class TsP(Generic[*Ts, P]): + pass + + object_to_expected_repr = { + MyCallable[P, T]: "MyCallable[~P, ~T]", + MyCallable[Concatenate[T1, P], T]: "MyCallable[typing.Concatenate[~T1, ~P], ~T]", + MyCallable[[], bool]: "MyCallable[[], bool]", + MyCallable[[int], bool]: "MyCallable[[int], bool]", + MyCallable[[int, str], bool]: "MyCallable[[int, str], bool]", + MyCallable[[int, list[int]], bool]: "MyCallable[[int, list[int]], bool]", + MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[*Ts, ~P], ~T]", + + DoubleSpec[P2, P, T]: "DoubleSpec[~P2, ~P, ~T]", + DoubleSpec[[int], [str], bool]: "DoubleSpec[[int], [str], bool]", + DoubleSpec[[int, int], [str, str], bool]: "DoubleSpec[[int, int], [str, str], bool]", + + TsP[*Ts, P]: "TsP[*Ts, ~P]", + TsP[int, str, list[int], []]: "TsP[int, str, list[int], []]", + TsP[int, [str, list[int]]]: "TsP[int, [str, list[int]]]", + + # These lines are just too long to fit: + MyCallable[Concatenate[*Ts, P], int][int, str, [bool, float]]: + "MyCallable[[int, str, bool, float], int]", + } + + for obj, expected_repr in object_to_expected_repr.items(): + with self.subTest(obj=obj, expected_repr=expected_repr): + self.assertRegex( + repr(obj), + fr"^{re.escape(MyCallable.__module__)}.*\.{re.escape(expected_repr)}$", + ) + def test_eq_1(self): self.assertEqual(Generic, Generic) self.assertEqual(Generic[T], Generic[T]) diff --git a/Lib/typing.py b/Lib/typing.py index 8d40e923bb1d08..ab334395676159 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -230,16 +230,17 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ - if isinstance(obj, types.GenericAlias): - return repr(obj) if isinstance(obj, type): if obj.__module__ == 'builtins': return obj.__qualname__ return f'{obj.__module__}.{obj.__qualname__}' if obj is ...: - return('...') + return '...' if isinstance(obj, types.FunctionType): return obj.__name__ + if isinstance(obj, tuple): + # Special case for `repr` of types with `ParamSpec`: + return '[' + ', '.join(_type_repr(t) for t in obj) + ']' return repr(obj) diff --git a/Misc/NEWS.d/next/Library/2023-03-13-12-05-55.gh-issue-102615.NcA_ZL.rst b/Misc/NEWS.d/next/Library/2023-03-13-12-05-55.gh-issue-102615.NcA_ZL.rst new file mode 100644 index 00000000000000..333068369bc4f7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-13-12-05-55.gh-issue-102615.NcA_ZL.rst @@ -0,0 +1,3 @@ +Typing: Improve the ``repr`` of generic aliases for classes generic over a +:class:`~typing.ParamSpec`. (Use square brackets to represent a parameter +list.) From afa6092ee4260bacf7bc11905466e4c3f8556cbb Mon Sep 17 00:00:00 2001 From: Max Bachmann Date: Wed, 15 Mar 2023 13:58:43 +0100 Subject: [PATCH 09/34] gh-102281: Fix potential nullptr dereference + use of uninitialized memory (gh-102282) --- .../2023-03-02-13-49-21.gh-issue-102281.QCuu2N.rst | 1 + Modules/getpath.c | 5 ++++- Python/fileutils.c | 6 +++++- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-02-13-49-21.gh-issue-102281.QCuu2N.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-02-13-49-21.gh-issue-102281.QCuu2N.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-02-13-49-21.gh-issue-102281.QCuu2N.rst new file mode 100644 index 00000000000000..b0269dd3d92bd5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-02-13-49-21.gh-issue-102281.QCuu2N.rst @@ -0,0 +1 @@ +Fix potential nullptr dereference and use of uninitialized memory in fileutils. Patch by Max Bachmann. diff --git a/Modules/getpath.c b/Modules/getpath.c index 2f20521592ce2e..237fe8c0c2c221 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -446,7 +446,10 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) if (s) { *s = L'\0'; } - path2 = _Py_normpath(_Py_join_relfile(path, resolved), -1); + path2 = _Py_join_relfile(path, resolved); + if (path2) { + path2 = _Py_normpath(path2, -1); + } PyMem_RawFree((void *)path); path = path2; } diff --git a/Python/fileutils.c b/Python/fileutils.c index 4ac759c45a3a1e..f48b626b444016 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2233,7 +2233,10 @@ _Py_join_relfile(const wchar_t *dirname, const wchar_t *relfile) } assert(wcslen(dirname) < MAXPATHLEN); assert(wcslen(relfile) < MAXPATHLEN - wcslen(dirname)); - join_relfile(filename, bufsize, dirname, relfile); + if (join_relfile(filename, bufsize, dirname, relfile) < 0) { + PyMem_RawFree(filename); + return NULL; + } return filename; } @@ -2271,6 +2274,7 @@ _Py_find_basename(const wchar_t *filename) wchar_t * _Py_normpath(wchar_t *path, Py_ssize_t size) { + assert(path != NULL); if (!path[0] || size == 0) { return path; } From 61b9ff35cbda0cc59816951a17de073968fc25c6 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Wed, 15 Mar 2023 16:10:03 +0100 Subject: [PATCH 10/34] gh-101100: Documenting --prefix and --exec-prefix. (GH-102695) Co-authored-by: Erlend E. Aasland --- Doc/c-api/init.rst | 2 +- Doc/c-api/intro.rst | 10 +++++----- Doc/library/sys.rst | 2 +- Doc/using/configure.rst | 16 ++++++++++++++++ Doc/using/unix.rst | 2 +- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index b50ee3b3803e29..38e324fb6409bc 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -513,7 +513,7 @@ Process-wide parameters program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The returned string points into static storage; the caller should not modify its value. This corresponds to the :makevar:`prefix` variable in the top-level - :file:`Makefile` and the ``--prefix`` argument to the :program:`configure` + :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` script at build time. The value is available to Python code as ``sys.prefix``. It is only useful on Unix. See also the next function. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 85eb24a495b640..acd4e033dfbc4b 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -78,19 +78,19 @@ used by extension writers. Structure member names do not have a reserved prefix. The header files are typically installed with Python. On Unix, these are located in the directories :file:`{prefix}/include/pythonversion/` and -:file:`{exec_prefix}/include/pythonversion/`, where :envvar:`prefix` and -:envvar:`exec_prefix` are defined by the corresponding parameters to Python's +:file:`{exec_prefix}/include/pythonversion/`, where :option:`prefix <--prefix>` and +:option:`exec_prefix <--exec-prefix>` are defined by the corresponding parameters to Python's :program:`configure` script and *version* is ``'%d.%d' % sys.version_info[:2]``. On Windows, the headers are installed -in :file:`{prefix}/include`, where :envvar:`prefix` is the installation +in :file:`{prefix}/include`, where ``prefix`` is the installation directory specified to the installer. To include the headers, place both directories (if different) on your compiler's search path for includes. Do *not* place the parent directories on the search path and then use ``#include ``; this will break on multi-platform builds since the platform independent headers under -:envvar:`prefix` include the platform specific headers from -:envvar:`exec_prefix`. +:option:`prefix <--prefix>` include the platform specific headers from +:option:`exec_prefix <--exec-prefix>`. C++ users should note that although the API is defined entirely using C, the header files properly declare the entry points to be ``extern "C"``. As a result, diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 23c5bbed0c6f9c..a53d4908783e15 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1323,7 +1323,7 @@ always available. A string giving the site-specific directory prefix where the platform independent Python files are installed; on Unix, the default is - ``'/usr/local'``. This can be set at build time with the ``--prefix`` + :file:`/usr/local`. This can be set at build time with the :option:`--prefix` argument to the :program:`configure` script. See :ref:`installation_paths` for derived paths. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 8936bd381c9d97..ce858ab2c8d79e 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -216,6 +216,22 @@ WebAssembly Options Install Options --------------- +.. cmdoption:: --prefix=PREFIX + + Install architecture-independent files in PREFIX. On Unix, it + defaults to :file:`/usr/local`. + + This value can be retrived at runtime using :data:`sys.prefix`. + + As an example, one can use ``--prefix="$HOME/.local/"`` to install + a Python in its home directory. + +.. cmdoption:: --exec-prefix=EPREFIX + + Install architecture-dependent files in EPREFIX, defaults to :option:`--prefix`. + + This value can be retrived at runtime using :data:`sys.exec_prefix`. + .. cmdoption:: --disable-test-modules Don't build nor install test modules, like the :mod:`test` package or the diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 24c02c99f871d5..067ff4cce5e48d 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -93,7 +93,7 @@ Python-related paths and files ============================== These are subject to difference depending on local installation conventions; -:envvar:`prefix` (``${prefix}``) and :envvar:`exec_prefix` (``${exec_prefix}``) +:option:`prefix <--prefix>` and :option:`exec_prefix <--exec-prefix>` are installation-dependent and should be interpreted as for GNU software; they may be the same. From 70185de1abfe428049a5c43d58fcb656b46db96c Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 15 Mar 2023 08:37:36 -0700 Subject: [PATCH 11/34] gh-102654: Insert #line directives in generated_cases.c.h (#102669) This behavior is optional, because in some extreme cases it may just make debugging harder. The tool defaults it to off, but it is on in Makefile.pre.in. Also note that this makes diffs to generated_cases.c.h noisier, since whenever you insert or delete a line in bytecodes.c, all subsequent #line directives will change. --- Makefile.pre.in | 1 + Python/generated_cases.c.h | 456 ++++++++++++++++++++++++ Tools/cases_generator/generate_cases.py | 89 ++++- 3 files changed, 530 insertions(+), 16 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 59762165c10036..8f13198e7e34b3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1485,6 +1485,7 @@ regen-cases: PYTHONPATH=$(srcdir)/Tools/cases_generator \ $(PYTHON_FOR_REGEN) \ $(srcdir)/Tools/cases_generator/generate_cases.py \ + --emit-line-directives \ -o $(srcdir)/Python/generated_cases.c.h.new \ -m $(srcdir)/Python/opcode_metadata.h.new \ $(srcdir)/Python/bytecodes.c diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b1dbb58d956367..26f6be049b3012 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,20 +8,24 @@ } TARGET(RESUME) { + #line 89 "Python/bytecodes.c" assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } + #line 18 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLOSURE) { PyObject *value; + #line 97 "Python/bytecodes.c" /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); + #line 29 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -29,9 +33,11 @@ TARGET(LOAD_FAST_CHECK) { PyObject *value; + #line 104 "Python/bytecodes.c" value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); + #line 41 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -39,9 +45,11 @@ TARGET(LOAD_FAST) { PyObject *value; + #line 110 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 53 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -50,8 +58,10 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; + #line 116 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); + #line 65 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -59,7 +69,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; + #line 121 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 75 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -69,17 +81,21 @@ PyObject *_tmp_2; { PyObject *value; + #line 110 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 89 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 110 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 99 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -93,16 +109,20 @@ PyObject *_tmp_2; { PyObject *value; + #line 110 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 117 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 116 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); + #line 126 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -115,14 +135,18 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; + #line 121 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 141 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 110 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 150 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -134,12 +158,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; + #line 121 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 164 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; + #line 121 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 171 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -150,16 +178,20 @@ PyObject *_tmp_2; { PyObject *value; + #line 116 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); + #line 185 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 110 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 195 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -170,6 +202,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; + #line 131 "Python/bytecodes.c" + #line 207 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -177,7 +211,9 @@ TARGET(PUSH_NULL) { PyObject *res; + #line 135 "Python/bytecodes.c" res = NULL; + #line 217 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -188,10 +224,14 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; + #line 131 "Python/bytecodes.c" + #line 229 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; + #line 131 "Python/bytecodes.c" + #line 235 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); @@ -201,9 +241,13 @@ TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 141 "Python/bytecodes.c" res = PyNumber_Negative(value); + #line 247 "Python/generated_cases.c.h" Py_DECREF(value); + #line 143 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; + #line 251 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -211,8 +255,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 147 "Python/bytecodes.c" int err = PyObject_IsTrue(value); + #line 261 "Python/generated_cases.c.h" Py_DECREF(value); + #line 149 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -221,6 +268,7 @@ res = Py_False; } Py_INCREF(res); + #line 272 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -228,9 +276,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 160 "Python/bytecodes.c" res = PyNumber_Invert(value); + #line 282 "Python/generated_cases.c.h" Py_DECREF(value); + #line 162 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; + #line 286 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -239,6 +291,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; + #line 179 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); @@ -247,6 +300,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; + #line 304 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -257,6 +311,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; + #line 190 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); @@ -264,6 +319,7 @@ double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); + #line 323 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -274,6 +330,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; + #line 200 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); @@ -282,6 +339,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; + #line 343 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -292,12 +350,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; + #line 211 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); + #line 361 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -308,6 +368,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 220 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -316,6 +377,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; + #line 381 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -325,6 +387,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; + #line 237 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -352,6 +415,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); + #line 419 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -360,6 +424,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; + #line 267 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -367,6 +432,7 @@ double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); + #line 436 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -377,6 +443,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; + #line 277 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -385,6 +452,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; + #line 456 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -397,6 +465,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; + #line 296 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -409,9 +478,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); + #line 482 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); + #line 309 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 487 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -423,6 +495,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; + #line 313 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -435,6 +508,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; + #line 512 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -445,6 +519,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; + #line 328 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -457,6 +532,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; + #line 536 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -465,6 +541,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; + #line 343 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -480,6 +557,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); + #line 561 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -490,6 +568,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; + #line 361 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -505,6 +584,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); + #line 588 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -515,6 +595,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; + #line 379 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -523,11 +604,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } + #line 608 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); + #line 388 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub + #line 615 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -541,6 +625,7 @@ PyObject *container = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t func_version = read_u16(&next_instr[3].cache); + #line 395 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -559,12 +644,15 @@ new_frame->localsplus[1] = sub; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); DISPATCH_INLINED(new_frame); + #line 648 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; + #line 416 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + #line 656 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -573,9 +661,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; + #line 421 "Python/bytecodes.c" int err = PySet_Add(set, v); + #line 667 "Python/generated_cases.c.h" Py_DECREF(v); + #line 423 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 671 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -588,6 +680,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); + #line 434 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); @@ -603,10 +696,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); + #line 700 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); + #line 450 "Python/bytecodes.c" if (err) goto pop_3_error; + #line 706 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -616,6 +712,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; + #line 454 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -633,6 +730,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); + #line 734 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -642,12 +740,14 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; + #line 474 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; + #line 751 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -656,11 +756,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; + #line 483 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); + #line 763 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); + #line 486 "Python/bytecodes.c" if (err) goto pop_2_error; + #line 768 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -668,10 +772,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 490 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); + #line 779 "Python/generated_cases.c.h" Py_DECREF(value); + #line 493 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; + #line 783 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -680,11 +788,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; + #line 497 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); + #line 795 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); + #line 500 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 800 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -692,6 +804,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); + #line 504 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -709,10 +822,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } + #line 826 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; + #line 524 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -724,10 +839,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; + #line 843 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; + #line 538 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -741,9 +858,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 862 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { + #line 554 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -758,11 +877,13 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 881 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; + #line 571 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -775,12 +896,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); + #line 900 "Python/generated_cases.c.h" Py_DECREF(obj); + #line 584 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); + #line 907 "Python/generated_cases.c.h" Py_DECREF(obj); + #line 589 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -793,6 +918,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } + #line 922 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -800,6 +926,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; + #line 604 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -843,6 +970,7 @@ } } + #line 974 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -853,13 +981,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; + #line 651 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } + #line 992 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 658 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -877,6 +1008,7 @@ if (iter == NULL) goto pop_1_error; + #line 1012 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -887,6 +1019,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; + #line 684 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -922,6 +1055,7 @@ assert(retval != NULL); } Py_DECREF(v); + #line 1059 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -930,6 +1064,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; + #line 722 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && @@ -945,10 +1080,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); + #line 1084 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; + #line 740 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -967,12 +1104,15 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 1108 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; + #line 761 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); + #line 1116 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -980,6 +1120,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); + #line 766 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -999,15 +1140,19 @@ PyObject *tb = PyException_GetTraceback(exc); _PyErr_Restore(tstate, typ, exc, tb); goto exception_unwind; + #line 1144 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; + #line 788 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + #line 1153 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); + #line 791 "Python/bytecodes.c" } else { Py_INCREF(exc); @@ -1016,6 +1161,7 @@ _PyErr_Restore(tstate, typ, exc, tb); goto exception_unwind; } + #line 1165 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1026,19 +1172,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; + #line 802 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + #line 1181 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); + #line 807 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } + #line 1192 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1047,7 +1197,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; + #line 816 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); + #line 1203 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1055,6 +1207,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; + #line 820 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1076,6 +1229,7 @@ if (true) goto error; } } + #line 1233 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1083,26 +1237,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; + #line 844 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); + #line 1248 "Python/generated_cases.c.h" Py_DECREF(v); + #line 851 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); + #line 1257 "Python/generated_cases.c.h" Py_DECREF(v); + #line 858 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 1261 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { + #line 862 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1119,6 +1280,7 @@ name); goto error; } + #line 1284 "Python/generated_cases.c.h" DISPATCH(); } @@ -1126,6 +1288,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; + #line 888 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1139,8 +1302,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); + #line 1306 "Python/generated_cases.c.h" Py_DECREF(seq); + #line 902 "Python/bytecodes.c" if (res == 0) goto pop_1_error; + #line 1310 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1150,12 +1316,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); + #line 906 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + #line 1327 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1166,6 +1334,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); + #line 916 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1173,6 +1342,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } + #line 1346 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1183,6 +1353,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); + #line 927 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1190,6 +1361,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } + #line 1365 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1199,11 +1371,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; + #line 938 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + #line 1379 "Python/generated_cases.c.h" Py_DECREF(seq); + #line 942 "Python/bytecodes.c" if (res == 0) goto pop_1_error; + #line 1383 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1214,6 +1390,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); + #line 953 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { assert(cframe.use_tracing == 0); @@ -1230,9 +1407,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); + #line 1411 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); + #line 970 "Python/bytecodes.c" if (err) goto pop_2_error; + #line 1416 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1240,25 +1420,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; + #line 974 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); + #line 1427 "Python/generated_cases.c.h" Py_DECREF(owner); + #line 977 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 1431 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; + #line 981 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); + #line 1441 "Python/generated_cases.c.h" Py_DECREF(v); + #line 984 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 1445 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { + #line 988 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1270,11 +1459,13 @@ } goto error; } + #line 1463 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; + #line 1002 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1333,6 +1524,7 @@ } } } + #line 1528 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1343,6 +1535,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; + #line 1069 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1395,6 +1588,7 @@ } } null = NULL; + #line 1592 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1408,6 +1602,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); + #line 1124 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1419,6 +1614,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; + #line 1618 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1433,6 +1629,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); + #line 1138 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); @@ -1447,6 +1644,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; + #line 1648 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1456,13 +1654,16 @@ } TARGET(DELETE_FAST) { + #line 1155 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); + #line 1662 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { + #line 1161 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1471,10 +1672,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); + #line 1676 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { + #line 1172 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1485,11 +1688,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); + #line 1692 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; + #line 1185 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1521,6 +1726,7 @@ } Py_INCREF(value); } + #line 1730 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1528,6 +1734,7 @@ TARGET(LOAD_DEREF) { PyObject *value; + #line 1219 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1535,6 +1742,7 @@ if (true) goto error; } Py_INCREF(value); + #line 1746 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1542,15 +1750,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; + #line 1229 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); + #line 1759 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { + #line 1236 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1561,17 +1772,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } + #line 1776 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; + #line 1249 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + #line 1785 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } + #line 1251 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } + #line 1791 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1581,8 +1797,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; + #line 1255 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } + #line 1804 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1592,8 +1810,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; + #line 1260 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } + #line 1817 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1603,6 +1823,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; + #line 1265 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1613,10 +1834,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } + #line 1838 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 1276 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); + #line 1844 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1625,9 +1849,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; + #line 1283 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); + #line 1855 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 1285 "Python/bytecodes.c" if (err < 0) goto pop_1_error; + #line 1859 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1635,6 +1863,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; + #line 1289 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -1649,6 +1878,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } + #line 1882 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -1658,6 +1888,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; + #line 1306 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -1665,10 +1896,13 @@ if (map == NULL) goto error; + #line 1900 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } + #line 1314 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } + #line 1906 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -1676,6 +1910,7 @@ } TARGET(SETUP_ANNOTATIONS) { + #line 1318 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -1715,6 +1950,7 @@ Py_DECREF(ann_dict); } } + #line 1954 "Python/generated_cases.c.h" DISPATCH(); } @@ -1722,6 +1958,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; + #line 1360 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1731,11 +1968,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); + #line 1972 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); + #line 1370 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } + #line 1979 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -1743,6 +1983,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; + #line 1374 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -1750,9 +1991,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } + #line 1995 "Python/generated_cases.c.h" Py_DECREF(update); + #line 1382 "Python/bytecodes.c" if (true) goto pop_1_error; } + #line 2000 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -1760,13 +2004,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; + #line 1388 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); + #line 2013 "Python/generated_cases.c.h" Py_DECREF(update); + #line 1393 "Python/bytecodes.c" if (true) goto pop_1_error; } + #line 2018 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -1776,11 +2024,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; + #line 1400 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + #line 2034 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -1792,6 +2042,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; + #line 1423 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1826,7 +2077,9 @@ NULL | meth | arg1 | ... | argN */ + #line 2081 "Python/generated_cases.c.h" Py_DECREF(owner); + #line 1458 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -1835,9 +2088,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); + #line 2092 "Python/generated_cases.c.h" Py_DECREF(owner); + #line 1467 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } + #line 2097 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -1851,6 +2107,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1472 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -1864,6 +2121,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2125 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1878,6 +2136,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1489 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; @@ -1891,6 +2150,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2154 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1905,6 +2165,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1506 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -1932,6 +2193,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2197 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1946,6 +2208,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1537 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -1956,6 +2219,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2223 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1970,6 +2234,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 1551 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); @@ -1982,6 +2247,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); + #line 2251 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1995,6 +2261,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); + #line 1567 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); @@ -2018,6 +2285,7 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); + #line 2289 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2025,6 +2293,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); + #line 1593 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2050,6 +2319,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); + #line 2323 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2057,6 +2327,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1621 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2075,6 +2346,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); + #line 2350 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2085,6 +2357,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); + #line 1642 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2124,6 +2397,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); + #line 2401 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2134,6 +2408,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1684 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -2144,6 +2419,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); + #line 2423 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2153,12 +2429,16 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 1697 "Python/bytecodes.c" STAT_INC(COMPARE_OP, deferred); assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); + #line 2437 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); + #line 1701 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 2442 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2169,6 +2449,7 @@ PREDICTED(COMPARE_AND_BRANCH); PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; + #line 1713 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2182,8 +2463,10 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); + #line 2467 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); + #line 1727 "Python/bytecodes.c" if (cond == NULL) goto pop_2_error; assert(next_instr[1].op.code == POP_JUMP_IF_FALSE || next_instr[1].op.code == POP_JUMP_IF_TRUE); @@ -2195,6 +2478,7 @@ if (jump_on_true == (err != 0)) { JUMPBY(offset); } + #line 2482 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 2; DISPATCH(); @@ -2203,6 +2487,7 @@ TARGET(COMPARE_AND_BRANCH_FLOAT) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; + #line 1741 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_AND_BRANCH); @@ -2217,6 +2502,7 @@ int offset = next_instr[1].op.arg; JUMPBY(offset); } + #line 2506 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 2; DISPATCH(); @@ -2225,6 +2511,7 @@ TARGET(COMPARE_AND_BRANCH_INT) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; + #line 1759 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_AND_BRANCH); @@ -2242,6 +2529,7 @@ int offset = next_instr[1].op.arg; JUMPBY(offset); } + #line 2533 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 2; DISPATCH(); @@ -2250,6 +2538,7 @@ TARGET(COMPARE_AND_BRANCH_STR) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; + #line 1780 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_AND_BRANCH); @@ -2265,6 +2554,7 @@ int offset = next_instr[1].op.arg; JUMPBY(offset); } + #line 2558 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 2; DISPATCH(); @@ -2274,10 +2564,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; + #line 1798 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; + #line 2570 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); + #line 1800 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); + #line 2575 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2287,11 +2581,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; + #line 1804 "Python/bytecodes.c" int res = PySequence_Contains(right, left); + #line 2587 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); + #line 1806 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); + #line 2593 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2302,9 +2600,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; + #line 1811 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { + #line 2606 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); + #line 1813 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2312,8 +2613,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); + #line 2617 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); + #line 1821 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2322,6 +2625,7 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } + #line 2629 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2331,15 +2635,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; + #line 1832 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { + #line 2642 "Python/generated_cases.c.h" Py_DECREF(right); + #line 1835 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); + #line 2649 "Python/generated_cases.c.h" Py_DECREF(right); + #line 1840 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); + #line 2653 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2348,11 +2658,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; + #line 1844 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); + #line 2665 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); + #line 1847 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 2670 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2361,23 +2675,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; + #line 1851 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; + #line 2683 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { + #line 1857 "Python/bytecodes.c" JUMPBY(oparg); + #line 2692 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); + #line 1861 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); + #line 2701 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2385,6 +2705,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; + #line 1867 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2394,7 +2715,9 @@ } else { int err = PyObject_IsTrue(cond); + #line 2719 "Python/generated_cases.c.h" Py_DECREF(cond); + #line 1877 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2402,12 +2725,14 @@ if (err < 0) goto pop_1_error; } } + #line 2729 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; + #line 1887 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2417,7 +2742,9 @@ } else { int err = PyObject_IsTrue(cond); + #line 2746 "Python/generated_cases.c.h" Py_DECREF(cond); + #line 1897 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2425,38 +2752,48 @@ if (err < 0) goto pop_1_error; } } + #line 2756 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; + #line 1907 "Python/bytecodes.c" if (!Py_IsNone(value)) { + #line 2765 "Python/generated_cases.c.h" Py_DECREF(value); + #line 1909 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } + #line 2773 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; + #line 1917 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { + #line 2786 "Python/generated_cases.c.h" Py_DECREF(value); + #line 1923 "Python/bytecodes.c" } + #line 2790 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_IF_FALSE_OR_POP) { PyObject *cond = stack_pointer[-1]; + #line 1927 "Python/bytecodes.c" bool jump = false; int err; if (Py_IsTrue(cond)) { @@ -2479,6 +2816,7 @@ goto error; } } + #line 2820 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW((jump ? 1 : 0)); DISPATCH(); @@ -2486,6 +2824,7 @@ TARGET(JUMP_IF_TRUE_OR_POP) { PyObject *cond = stack_pointer[-1]; + #line 1952 "Python/bytecodes.c" bool jump = false; int err; if (Py_IsFalse(cond)) { @@ -2508,29 +2847,34 @@ goto error; } } + #line 2851 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW((jump ? 1 : 0)); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + #line 1977 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); + #line 2865 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; + #line 1986 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; + #line 2878 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2541,13 +2885,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; + #line 1994 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); + #line 2894 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); + #line 1999 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2555,6 +2902,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } + #line 2906 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2563,8 +2911,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; + #line 2009 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); + #line 2918 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2574,8 +2924,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; + #line 2015 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); + #line 2931 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2586,9 +2938,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; + #line 2021 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; + #line 2946 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2597,10 +2951,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; + #line 2027 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); + #line 2958 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 2030 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; + #line 2962 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2608,6 +2966,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; + #line 2034 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -2630,8 +2989,11 @@ if (iter == NULL) { goto error; } + #line 2993 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 2057 "Python/bytecodes.c" } + #line 2997 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -2642,6 +3004,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2076 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2674,6 +3037,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator + #line 3041 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2683,6 +3047,7 @@ TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2111 "Python/bytecodes.c" assert(cframe.use_tracing == 0); DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; @@ -2703,6 +3068,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator + #line 3072 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2712,6 +3078,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2134 "Python/bytecodes.c" assert(cframe.use_tracing == 0); _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); @@ -2732,6 +3099,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator + #line 3103 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2741,6 +3109,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2157 "Python/bytecodes.c" assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); @@ -2759,6 +3128,7 @@ if (next == NULL) { goto error; } + #line 3132 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2767,6 +3137,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; + #line 2178 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -2781,12 +3152,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); assert(next_instr->op.code == END_FOR); DISPATCH_INLINED(gen_frame); + #line 3156 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; + #line 2195 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -2809,13 +3182,16 @@ Py_DECREF(enter); goto error; } + #line 3186 "Python/generated_cases.c.h" Py_DECREF(mgr); + #line 2218 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } + #line 3195 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -2827,6 +3203,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; + #line 2228 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -2852,13 +3229,16 @@ Py_DECREF(enter); goto error; } + #line 3233 "Python/generated_cases.c.h" Py_DECREF(mgr); + #line 2254 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } + #line 3242 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -2870,6 +3250,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; + #line 2263 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -2890,6 +3271,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; + #line 3275 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -2898,6 +3280,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; + #line 2286 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -2907,6 +3290,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); + #line 3294 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -2920,6 +3304,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 2298 "Python/bytecodes.c" /* Cached method object */ assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); @@ -2937,6 +3322,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); + #line 3326 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2950,6 +3336,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 2318 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2960,6 +3347,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); + #line 3351 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2973,6 +3361,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 2331 "Python/bytecodes.c" assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2987,6 +3376,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); + #line 3380 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2995,9 +3385,11 @@ } TARGET(KW_NAMES) { + #line 2348 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); + #line 3393 "Python/generated_cases.c.h" DISPATCH(); } @@ -3008,6 +3400,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2384 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3079,6 +3472,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3476 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3090,6 +3484,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; + #line 2462 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3099,6 +3494,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + #line 3498 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3107,6 +3503,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); + #line 2474 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3131,6 +3528,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); + #line 3532 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3139,6 +3537,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); + #line 2501 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3168,6 +3567,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); + #line 3571 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3175,6 +3575,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2533 "Python/bytecodes.c" assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); @@ -3185,6 +3586,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable + #line 3590 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3197,6 +3599,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2546 "Python/bytecodes.c" assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); @@ -3208,6 +3611,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3615 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3221,6 +3625,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2561 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3231,6 +3636,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3640 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3244,6 +3650,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2575 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3265,6 +3672,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3676 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3278,6 +3686,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2600 "Python/bytecodes.c" assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); @@ -3306,6 +3715,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3719 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3319,6 +3729,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2632 "Python/bytecodes.c" assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); @@ -3351,6 +3762,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ + #line 3766 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3364,6 +3776,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2668 "Python/bytecodes.c" assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; @@ -3396,6 +3809,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3813 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3409,6 +3823,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2704 "Python/bytecodes.c" assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ @@ -3434,6 +3849,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3853 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3446,6 +3862,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2732 "Python/bytecodes.c" assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ @@ -3473,6 +3890,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3894 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3484,6 +3902,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; + #line 2763 "Python/bytecodes.c" assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); @@ -3502,12 +3921,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); + #line 3925 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2784 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3538,6 +3959,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3963 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3550,6 +3972,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2818 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3578,6 +4001,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4005 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3590,6 +4014,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2850 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -3618,6 +4043,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4047 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3630,6 +4056,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2882 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3657,6 +4084,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4088 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3671,6 +4099,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; + #line 2913 "Python/bytecodes.c" if (oparg & 1) { // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. @@ -3689,12 +4118,15 @@ assert(PyTuple_CheckExact(callargs)); result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + #line 4122 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); + #line 2932 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } + #line 4130 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -3709,6 +4141,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; + #line 2943 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -3737,12 +4170,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; + #line 4174 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { + #line 2974 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -3763,6 +4198,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; + #line 4202 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -3770,11 +4206,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; + #line 2997 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); + #line 4212 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); + #line 2999 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } + #line 4218 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -3785,6 +4225,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; + #line 3003 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -3826,10 +4267,13 @@ } else { /* Actually call format(). */ result = PyObject_Format(value, fmt_spec); + #line 4271 "Python/generated_cases.c.h" Py_DECREF(value); Py_XDECREF(fmt_spec); + #line 3045 "Python/bytecodes.c" if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } } + #line 4277 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -3838,8 +4282,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; + #line 3050 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); + #line 4289 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -3851,6 +4297,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; + #line 3055 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3866,9 +4313,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); + #line 4317 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); + #line 3071 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 4322 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -3878,21 +4328,27 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; + #line 3076 "Python/bytecodes.c" assert(oparg >= 2); + #line 4334 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(EXTENDED_ARG) { + #line 3080 "Python/bytecodes.c" assert(oparg); assert(cframe.use_tracing == 0); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); + #line 4348 "Python/generated_cases.c.h" } TARGET(CACHE) { + #line 3089 "Python/bytecodes.c" Py_UNREACHABLE(); + #line 4354 "Python/generated_cases.c.h" } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 25cf75e4c1c490..b16c927a4a2870 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -13,6 +13,7 @@ import sys import typing +import lexer as lx import parser from parser import StackEffect @@ -43,6 +44,9 @@ arg_parser.add_argument( "-m", "--metadata", type=str, help="Generated metadata", default=DEFAULT_METADATA_OUTPUT ) +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) arg_parser.add_argument( "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" ) @@ -105,13 +109,34 @@ class Formatter: stream: typing.TextIO prefix: str - - def __init__(self, stream: typing.TextIO, indent: int) -> None: + emit_line_directives: bool = False + lineno: int # Next line number, 1-based + filename: str # Slightly improved stream.filename + nominal_lineno: int + nominal_filename: str + + def __init__( + self, stream: typing.TextIO, indent: int, emit_line_directives: bool = False + ) -> None: self.stream = stream self.prefix = " " * indent + self.emit_line_directives = emit_line_directives + self.lineno = 1 + # Make filename more user-friendly and less platform-specific + filename = self.stream.name.replace("\\", "/") + if filename.startswith("./"): + filename = filename[2:] + if filename.endswith(".new"): + filename = filename[:-4] + self.filename = filename + self.nominal_lineno = 1 + self.nominal_filename = filename def write_raw(self, s: str) -> None: self.stream.write(s) + newlines = s.count("\n") + self.lineno += newlines + self.nominal_lineno += newlines def emit(self, arg: str) -> None: if arg: @@ -119,6 +144,17 @@ def emit(self, arg: str) -> None: else: self.write_raw("\n") + def set_lineno(self, lineno: int, filename: str) -> None: + if self.emit_line_directives: + if lineno != self.nominal_lineno or filename != self.nominal_filename: + self.emit(f'#line {lineno} "{filename}"') + self.nominal_lineno = lineno + self.nominal_filename = filename + + def reset_lineno(self) -> None: + if self.lineno != self.nominal_lineno or self.filename != self.nominal_filename: + self.set_lineno(self.lineno + 1, self.filename) + @contextlib.contextmanager def indent(self): self.prefix += " " @@ -199,6 +235,7 @@ class Instruction: block: parser.Block block_text: list[str] # Block.text, less curlies, less PREDICT() calls predictions: list[str] # Prediction targets (instruction names) + block_line: int # First line of block in original code # Computed by constructor always_exits: bool @@ -223,7 +260,7 @@ def __init__(self, inst: parser.InstDef): self.kind = inst.kind self.name = inst.name self.block = inst.block - self.block_text, self.check_eval_breaker, self.predictions = \ + self.block_text, self.check_eval_breaker, self.predictions, self.block_line = \ extract_block_text(self.block) self.always_exits = always_exits(self.block_text) self.cache_effects = [ @@ -388,7 +425,13 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None assert dedent <= 0 extra = " " * -dedent names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"}) + offset = 0 + context = self.block.context + assert context != None + filename = context.owner.filename for line in self.block_text: + out.set_lineno(self.block_line + offset, filename) + offset += 1 if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line): space, cond, label = m.groups() space = extra + space @@ -416,6 +459,7 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None out.write_raw(f"{space}if ({cond}) goto {label};\n") elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): if not self.register: + out.reset_lineno() space = extra + m.group(1) for ieff in self.input_effects: if ieff.name in names_to_skip: @@ -431,6 +475,7 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None out.write_raw(f"{space}Py_{decref}({ieff.name});\n") else: out.write_raw(extra + line) + out.reset_lineno() InstructionOrCacheEffect = Instruction | parser.CacheEffect @@ -501,6 +546,7 @@ class Analyzer: output_filename: str metadata_filename: str errors: int = 0 + emit_line_directives: bool = False def __init__(self, input_filenames: list[str], output_filename: str, metadata_filename: str): """Read the input file.""" @@ -561,6 +607,10 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: with open(filename) as file: src = file.read() + # Make filename more user-friendly and less platform-specific + filename = filename.replace("\\", "/") + if filename.startswith("./"): + filename = filename[2:] psr = parser.Parser(src, filename=filename) # Skip until begin marker @@ -972,13 +1022,14 @@ def write_metadata(self) -> None: format_enums = [INSTR_FMT_PREFIX + format for format in sorted(all_formats)] with open(self.metadata_filename, "w") as f: + # Create formatter + self.out = Formatter(f, 0) + # Write provenance header - f.write(f"// This file is generated by {THIS}\n") - f.write(self.from_source_files()) - f.write(f"// Do not edit!\n") + self.out.write_raw(f"// This file is generated by {THIS}\n") + self.out.write_raw(self.from_source_files()) + self.out.write_raw(f"// Do not edit!\n") - # Create formatter; the rest of the code uses this - self.out = Formatter(f, 0) self.write_stack_effect_functions() @@ -1053,13 +1104,13 @@ def write_metadata_for_macro(self, mac: MacroInstruction) -> None: def write_instructions(self) -> None: """Write instructions to output file.""" with open(self.output_filename, "w") as f: - # Write provenance header - f.write(f"// This file is generated by {THIS}\n") - f.write(self.from_source_files()) - f.write(f"// Do not edit!\n") + # Create formatter + self.out = Formatter(f, 8, self.emit_line_directives) - # Create formatter; the rest of the code uses this - self.out = Formatter(f, 8) + # Write provenance header + self.out.write_raw(f"// This file is generated by {THIS}\n") + self.out.write_raw(self.from_source_files()) + self.out.write_raw(f"// Do not edit!\n") # Write and count instructions of all kinds n_instrs = 0 @@ -1179,13 +1230,16 @@ def wrap_super_or_macro(self, up: SuperOrMacroInstruction): self.out.emit(f"DISPATCH();") -def extract_block_text(block: parser.Block) -> tuple[list[str], bool, list[str]]: +def extract_block_text(block: parser.Block) -> tuple[list[str], bool, list[str], int]: # Get lines of text with proper dedent blocklines = block.text.splitlines(True) + first_token: lx.Token = block.tokens[0] # IndexError means the context is broken + block_line = first_token.begin[0] # Remove blank lines from both ends while blocklines and not blocklines[0].strip(): blocklines.pop(0) + block_line += 1 while blocklines and not blocklines[-1].strip(): blocklines.pop() @@ -1194,6 +1248,7 @@ def extract_block_text(block: parser.Block) -> tuple[list[str], bool, list[str]] assert blocklines and blocklines[-1].strip() == "}" blocklines.pop() blocklines.pop(0) + block_line += 1 # Remove trailing blank lines while blocklines and not blocklines[-1].strip(): @@ -1213,7 +1268,7 @@ def extract_block_text(block: parser.Block) -> tuple[list[str], bool, list[str]] predictions.insert(0, m.group(1)) blocklines.pop() - return blocklines, check_eval_breaker, predictions + return blocklines, check_eval_breaker, predictions, block_line def always_exits(lines: list[str]) -> bool: @@ -1250,6 +1305,8 @@ def main(): if len(args.input) == 0: args.input.append(DEFAULT_INPUT) a = Analyzer(args.input, args.output, args.metadata) # Raises OSError if input unreadable + if args.emit_line_directives: + a.emit_line_directives = True a.parse() # Raises SyntaxError on failure a.analyze() # Prints messages and sets a.errors on failure if a.errors: From 215007b57619978243b5d5e4d0884f777e8b2715 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 15 Mar 2023 18:19:07 +0000 Subject: [PATCH 12/34] Exclude `ceval.c` from the C-analyzer tool (#102735) The "check if generated files are up to date" CI check appears to be currently failing on all PRs (but not on pushes to main) See, for example: - https://github.com/python/cpython/pull/94468 - https://github.com/python/cpython/pull/94468 - https://github.com/python/cpython/pull/102731 This appears to be because the C-analyzer tool doesn't like the `#line` directives introduced in https://github.com/python/cpython/commit/70185de1abfe428049a5c43d58fcb656b46db96c. I'm advised by the message printed to the terminal in https://github.com/python/cpython/actions/runs/4428706945/jobs/7768216988#step:14:84 that this is the appropriate short-term fix! --- Tools/c-analyzer/cpython/_parser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 6da4fbbf4224f1..a2911e030ffee1 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -91,10 +91,15 @@ def clean_lines(text): # XXX Fix the parser. EXCLUDED += clean_lines(''' # The tool should be able to parse these... + # The problem with xmlparse.c is that something # has gone wrong where # we handle "maybe inline actual" # in Tools/c-analyzer/c_parser/parser/_global.py. Modules/expat/xmlparse.c + +# The parser doesn't like the #line directives +# that originate from generated_cases.c.h +Python/ceval.c ''') INCL_DIRS = clean_lines(''' From 00d1ef73d6799142f90d8e00f3dfcf5d86e6cad8 Mon Sep 17 00:00:00 2001 From: Martin Breuss Date: Wed, 15 Mar 2023 20:18:18 +0100 Subject: [PATCH 13/34] Fix typo in code comment (#102726) --- Lib/test/test_traceback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 92c5a000585855..7ef93b3f0ac332 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -2987,7 +2987,7 @@ class MyClass: def test_getattr_suggestions_do_not_trigger_for_big_dicts(self): class A: blech = None - # A class with a very big __dict__ will not be consider + # A class with a very big __dict__ will not be considered # for suggestions. for index in range(2000): setattr(A, f"index_{index}", None) From 0a22aa0528a4ff590854996b8854e9a79120987a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 15 Mar 2023 15:15:23 -0500 Subject: [PATCH 14/34] Simplify and speed-up math.hypot() and math.dist() (GH-102734) --- Modules/mathmodule.c | 293 ++++++++++++++++++++----------------------- 1 file changed, 139 insertions(+), 154 deletions(-) diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 544560e8322c72..ae9e3211c072d8 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -92,6 +92,113 @@ get_math_module_state(PyObject *module) return (math_module_state *)state; } +/* +Double and triple length extended precision algorithms from: + + Accurate Sum and Dot Product + by Takeshi Ogita, Siegfried M. Rump, and Shin’Ichi Oishi + https://doi.org/10.1137/030601818 + https://www.tuhh.de/ti3/paper/rump/OgRuOi05.pdf + +*/ + +typedef struct{ double hi; double lo; } DoubleLength; + +static DoubleLength +dl_fast_sum(double a, double b) +{ + /* Algorithm 1.1. Compensated summation of two floating point numbers. */ + assert(fabs(a) >= fabs(b)); + double x = a + b; + double y = (a - x) + b; + return (DoubleLength) {x, y}; +} + +static DoubleLength +dl_sum(double a, double b) +{ + /* Algorithm 3.1 Error-free transformation of the sum */ + double x = a + b; + double z = x - a; + double y = (a - (x - z)) + (b - z); + return (DoubleLength) {x, y}; +} + +#ifndef UNRELIABLE_FMA + +static DoubleLength +dl_mul(double x, double y) +{ + /* Algorithm 3.5. Error-free transformation of a product */ + double z = x * y; + double zz = fma(x, y, -z); + return (DoubleLength) {z, zz}; +} + +#else + +/* + The default implementation of dl_mul() depends on the C math library + having an accurate fma() function as required by § 7.12.13.1 of the + C99 standard. + + The UNRELIABLE_FMA option is provided as a slower but accurate + alternative for builds where the fma() function is found wanting. + The speed penalty may be modest (17% slower on an Apple M1 Max), + so don't hesitate to enable this build option. + + The algorithms are from the T. J. Dekker paper: + A Floating-Point Technique for Extending the Available Precision + https://csclub.uwaterloo.ca/~pbarfuss/dekker1971.pdf +*/ + +static DoubleLength +dl_split(double x) { + // Dekker (5.5) and (5.6). + double t = x * 134217729.0; // Veltkamp constant = 2.0 ** 27 + 1 + double hi = t - (t - x); + double lo = x - hi; + return (DoubleLength) {hi, lo}; +} + +static DoubleLength +dl_mul(double x, double y) +{ + // Dekker (5.12) and mul12() + DoubleLength xx = dl_split(x); + DoubleLength yy = dl_split(y); + double p = xx.hi * yy.hi; + double q = xx.hi * yy.lo + xx.lo * yy.hi; + double z = p + q; + double zz = p - z + q + xx.lo * yy.lo; + return (DoubleLength) {z, zz}; +} + +#endif + +typedef struct { double hi; double lo; double tiny; } TripleLength; + +static const TripleLength tl_zero = {0.0, 0.0, 0.0}; + +static TripleLength +tl_fma(double x, double y, TripleLength total) +{ + /* Algorithm 5.10 with SumKVert for K=3 */ + DoubleLength pr = dl_mul(x, y); + DoubleLength sm = dl_sum(total.hi, pr.hi); + DoubleLength r1 = dl_sum(total.lo, pr.lo); + DoubleLength r2 = dl_sum(r1.hi, sm.lo); + return (TripleLength) {sm.hi, r2.hi, total.tiny + r1.lo + r2.lo}; +} + +static double +tl_to_d(TripleLength total) +{ + DoubleLength last = dl_sum(total.lo, total.hi); + return total.tiny + last.lo + last.hi; +} + + /* sin(pi*x), giving accurate results for all finite x (especially x integral or close to an integer). This is here for use in the @@ -2301,6 +2408,7 @@ that are almost always correctly rounded, four techniques are used: * lossless scaling using a power-of-two scaling factor * accurate squaring using Veltkamp-Dekker splitting [1] + or an equivalent with an fma() call * compensated summation using a variant of the Neumaier algorithm [2] * differential correction of the square root [3] @@ -2359,14 +2467,21 @@ algorithm, effectively doubling the number of accurate bits. This technique is used in Dekker's SQRT2 algorithm and again in Borges' ALGORITHM 4 and 5. -Without proof for all cases, hypot() cannot claim to be always -correctly rounded. However for n <= 1000, prior to the final addition -that rounds the overall result, the internal accuracy of "h" together -with its correction of "x / (2.0 * h)" is at least 100 bits. [6] -Also, hypot() was tested against a Decimal implementation with -prec=300. After 100 million trials, no incorrectly rounded examples -were found. In addition, perfect commutativity (all permutations are -exactly equal) was verified for 1 billion random inputs with n=5. [7] +The hypot() function is faithfully rounded (less than 1 ulp error) +and usually correctly rounded (within 1/2 ulp). The squaring +step is exact. The Neumaier summation computes as if in doubled +precision (106 bits) and has the advantage that its input squares +are non-negative so that the condition number of the sum is one. +The square root with a differential correction is likewise computed +as if in double precision. + +For n <= 1000, prior to the final addition that rounds the overall +result, the internal accuracy of "h" together with its correction of +"x / (2.0 * h)" is at least 100 bits. [6] Also, hypot() was tested +against a Decimal implementation with prec=300. After 100 million +trials, no incorrectly rounded examples were found. In addition, +perfect commutativity (all permutations are exactly equal) was +verified for 1 billion random inputs with n=5. [7] References: @@ -2383,9 +2498,8 @@ exactly equal) was verified for 1 billion random inputs with n=5. [7] static inline double vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) { - const double T27 = 134217729.0; /* ldexp(1.0, 27) + 1.0) */ - double x, scale, oldcsum, csum = 1.0, frac1 = 0.0, frac2 = 0.0, frac3 = 0.0; - double t, hi, lo, h; + double x, h, scale, oldcsum, csum = 1.0, frac1 = 0.0, frac2 = 0.0; + DoubleLength pr, sm; int max_e; Py_ssize_t i; @@ -2410,54 +2524,21 @@ vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) x *= scale; assert(fabs(x) < 1.0); - t = x * T27; - hi = t - (t - x); - lo = x - hi; - assert(hi + lo == x); - - x = hi * hi; - assert(x <= 1.0); - assert(fabs(csum) >= fabs(x)); - oldcsum = csum; - csum += x; - frac1 += (oldcsum - csum) + x; - - x = 2.0 * hi * lo; - assert(fabs(csum) >= fabs(x)); - oldcsum = csum; - csum += x; - frac2 += (oldcsum - csum) + x; - - assert(csum + lo * lo == csum); - frac3 += lo * lo; - } - h = sqrt(csum - 1.0 + (frac1 + frac2 + frac3)); - - x = h; - t = x * T27; - hi = t - (t - x); - lo = x - hi; - assert (hi + lo == x); + pr = dl_mul(x, x); + assert(pr.hi <= 1.0); - x = -hi * hi; - assert(fabs(csum) >= fabs(x)); - oldcsum = csum; - csum += x; - frac1 += (oldcsum - csum) + x; - - x = -2.0 * hi * lo; - assert(fabs(csum) >= fabs(x)); - oldcsum = csum; - csum += x; - frac2 += (oldcsum - csum) + x; - - x = -lo * lo; - assert(fabs(csum) >= fabs(x)); - oldcsum = csum; - csum += x; - frac3 += (oldcsum - csum) + x; - - x = csum - 1.0 + (frac1 + frac2 + frac3); + sm = dl_fast_sum(csum, pr.hi); + csum = sm.hi; + frac1 += pr.lo; + frac2 += sm.lo; + } + h = sqrt(csum - 1.0 + (frac1 + frac2)); + pr = dl_mul(-h, h); + sm = dl_fast_sum(csum, pr.hi); + csum = sm.hi; + frac1 += pr.lo; + frac2 += sm.lo; + x = csum - 1.0 + (frac1 + frac2); return (h + x / (2.0 * h)) / scale; } /* When max_e < -1023, ldexp(1.0, -max_e) overflows. @@ -2646,102 +2727,6 @@ long_add_would_overflow(long a, long b) return (a > 0) ? (b > LONG_MAX - a) : (b < LONG_MIN - a); } -/* -Double and triple length extended precision algorithms from: - - Accurate Sum and Dot Product - by Takeshi Ogita, Siegfried M. Rump, and Shin’Ichi Oishi - https://doi.org/10.1137/030601818 - https://www.tuhh.de/ti3/paper/rump/OgRuOi05.pdf - -*/ - -typedef struct{ double hi; double lo; } DoubleLength; - -static DoubleLength -dl_sum(double a, double b) -{ - /* Algorithm 3.1 Error-free transformation of the sum */ - double x = a + b; - double z = x - a; - double y = (a - (x - z)) + (b - z); - return (DoubleLength) {x, y}; -} - -#ifndef UNRELIABLE_FMA - -static DoubleLength -dl_mul(double x, double y) -{ - /* Algorithm 3.5. Error-free transformation of a product */ - double z = x * y; - double zz = fma(x, y, -z); - return (DoubleLength) {z, zz}; -} - -#else - -/* - The default implementation of dl_mul() depends on the C math library - having an accurate fma() function as required by § 7.12.13.1 of the - C99 standard. - - The UNRELIABLE_FMA option is provided as a slower but accurate - alternative for builds where the fma() function is found wanting. - The speed penalty may be modest (17% slower on an Apple M1 Max), - so don't hesitate to enable this build option. - - The algorithms are from the T. J. Dekker paper: - A Floating-Point Technique for Extending the Available Precision - https://csclub.uwaterloo.ca/~pbarfuss/dekker1971.pdf -*/ - -static DoubleLength -dl_split(double x) { - // Dekker (5.5) and (5.6). - double t = x * 134217729.0; // Veltkamp constant = 2.0 ** 27 + 1 - double hi = t - (t - x); - double lo = x - hi; - return (DoubleLength) {hi, lo}; -} - -static DoubleLength -dl_mul(double x, double y) -{ - // Dekker (5.12) and mul12() - DoubleLength xx = dl_split(x); - DoubleLength yy = dl_split(y); - double p = xx.hi * yy.hi; - double q = xx.hi * yy.lo + xx.lo * yy.hi; - double z = p + q; - double zz = p - z + q + xx.lo * yy.lo; - return (DoubleLength) {z, zz}; -} - -#endif - -typedef struct { double hi; double lo; double tiny; } TripleLength; - -static const TripleLength tl_zero = {0.0, 0.0, 0.0}; - -static TripleLength -tl_fma(double x, double y, TripleLength total) -{ - /* Algorithm 5.10 with SumKVert for K=3 */ - DoubleLength pr = dl_mul(x, y); - DoubleLength sm = dl_sum(total.hi, pr.hi); - DoubleLength r1 = dl_sum(total.lo, pr.lo); - DoubleLength r2 = dl_sum(r1.hi, sm.lo); - return (TripleLength) {sm.hi, r2.hi, total.tiny + r1.lo + r2.lo}; -} - -static double -tl_to_d(TripleLength total) -{ - DoubleLength last = dl_sum(total.lo, total.hi); - return total.tiny + last.lo + last.hi; -} - /*[clinic input] math.sumprod From 675b97a6ab483573f07ce6e0b79b0bc370ab76c9 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 15 Mar 2023 21:25:31 +0000 Subject: [PATCH 15/34] gh-102738: remove from cases generator the code related to register instructions (#102739) --- Python/opcode_metadata.h | 344 ++++++++++++------------ Tools/cases_generator/generate_cases.py | 190 +++++-------- 2 files changed, 234 insertions(+), 300 deletions(-) diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 93f3c76c5b8240..ee760ab78af217 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -707,12 +707,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum Direction { DIR_NONE, DIR_READ, DIR_WRITE }; enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC0, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; struct opcode_metadata { - enum Direction dir_op1; - enum Direction dir_op2; - enum Direction dir_op3; bool valid_entry; enum InstructionFormat instr_format; }; @@ -721,175 +717,175 @@ struct opcode_metadata { extern const struct opcode_metadata _PyOpcode_opcode_metadata[256]; #else const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { - [NOP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [RESUME] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_CLOSURE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_FAST_CHECK] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_CONST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [STORE_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_FAST__LOAD_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBIB }, - [LOAD_FAST__LOAD_CONST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBIB }, - [STORE_FAST__LOAD_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBIB }, - [STORE_FAST__STORE_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBIB }, - [LOAD_CONST__LOAD_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBIB }, - [POP_TOP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [PUSH_NULL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [END_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [UNARY_NEGATIVE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [UNARY_NOT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [UNARY_INVERT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [BINARY_OP_MULTIPLY_INT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, - [BINARY_OP_MULTIPLY_FLOAT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, - [BINARY_OP_SUBTRACT_INT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, - [BINARY_OP_SUBTRACT_FLOAT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, - [BINARY_OP_ADD_UNICODE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [BINARY_OP_ADD_FLOAT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, - [BINARY_OP_ADD_INT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, - [BINARY_SUBSCR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, - [BINARY_SLICE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [STORE_SLICE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [BINARY_SUBSCR_LIST_INT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, - [BINARY_SUBSCR_TUPLE_INT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, - [BINARY_SUBSCR_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, - [BINARY_SUBSCR_GETITEM] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, - [LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [SET_ADD] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [STORE_SUBSCR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, - [STORE_SUBSCR_LIST_INT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, - [STORE_SUBSCR_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, - [DELETE_SUBSCR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [CALL_INTRINSIC_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_INTRINSIC_2] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [RAISE_VARARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [INTERPRETER_EXIT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [RETURN_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [RETURN_CONST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [GET_AITER] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [GET_ANEXT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [GET_AWAITABLE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [SEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [SEND_GEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [YIELD_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [POP_EXCEPT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [RERAISE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [END_ASYNC_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [CLEANUP_THROW] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [LOAD_ASSERTION_ERROR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [LOAD_BUILD_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [STORE_NAME] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [DELETE_NAME] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [UNPACK_SEQUENCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [UNPACK_SEQUENCE_TUPLE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [UNPACK_SEQUENCE_LIST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [UNPACK_EX] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [STORE_ATTR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [DELETE_ATTR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [STORE_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [DELETE_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_NAME] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_GLOBAL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [LOAD_GLOBAL_MODULE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [LOAD_GLOBAL_BUILTIN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [DELETE_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [MAKE_CELL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [DELETE_DEREF] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_CLASSDEREF] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_DEREF] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [STORE_DEREF] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [COPY_FREE_VARS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [BUILD_STRING] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [BUILD_TUPLE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [BUILD_LIST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LIST_EXTEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [SET_UPDATE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [BUILD_SET] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [BUILD_MAP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [SETUP_ANNOTATIONS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [BUILD_CONST_KEY_MAP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [DICT_UPDATE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [DICT_MERGE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [MAP_ADD] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [LOAD_ATTR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_INSTANCE_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_MODULE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_WITH_HINT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_SLOT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_PROPERTY] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [STORE_ATTR_INSTANCE_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, - [STORE_ATTR_WITH_HINT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [STORE_ATTR_SLOT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, - [COMPARE_OP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [COMPARE_AND_BRANCH] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, - [COMPARE_AND_BRANCH_FLOAT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, - [COMPARE_AND_BRANCH_INT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, - [COMPARE_AND_BRANCH_STR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, - [IS_OP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CONTAINS_OP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CHECK_EG_MATCH] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [CHECK_EXC_MATCH] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [IMPORT_NAME] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [IMPORT_FROM] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [JUMP_FORWARD] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [JUMP_BACKWARD] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [POP_JUMP_IF_FALSE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [POP_JUMP_IF_TRUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [POP_JUMP_IF_NOT_NONE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [POP_JUMP_IF_NONE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [JUMP_IF_FALSE_OR_POP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [JUMP_IF_TRUE_OR_POP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [JUMP_BACKWARD_NO_INTERRUPT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [GET_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [MATCH_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [MATCH_MAPPING] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [MATCH_SEQUENCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [MATCH_KEYS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [GET_ITER] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [GET_YIELD_FROM_ITER] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [FOR_ITER] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [FOR_ITER_LIST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [FOR_ITER_TUPLE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [FOR_ITER_RANGE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [FOR_ITER_GEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [BEFORE_ASYNC_WITH] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [BEFORE_WITH] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [WITH_EXCEPT_START] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [PUSH_EXC_INFO] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_METHOD_NO_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, - [CALL_FUNCTION_EX] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [MAKE_FUNCTION] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [RETURN_GENERATOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, - [BUILD_SLICE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [FORMAT_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [COPY] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [BINARY_OP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [SWAP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [EXTENDED_ARG] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CACHE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, + [NOP] = { true, INSTR_FMT_IX }, + [RESUME] = { true, INSTR_FMT_IB }, + [LOAD_CLOSURE] = { true, INSTR_FMT_IB }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB }, + [LOAD_FAST] = { true, INSTR_FMT_IB }, + [LOAD_CONST] = { true, INSTR_FMT_IB }, + [STORE_FAST] = { true, INSTR_FMT_IB }, + [LOAD_FAST__LOAD_FAST] = { true, INSTR_FMT_IBIB }, + [LOAD_FAST__LOAD_CONST] = { true, INSTR_FMT_IBIB }, + [STORE_FAST__LOAD_FAST] = { true, INSTR_FMT_IBIB }, + [STORE_FAST__STORE_FAST] = { true, INSTR_FMT_IBIB }, + [LOAD_CONST__LOAD_FAST] = { true, INSTR_FMT_IBIB }, + [POP_TOP] = { true, INSTR_FMT_IX }, + [PUSH_NULL] = { true, INSTR_FMT_IX }, + [END_FOR] = { true, INSTR_FMT_IB }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX }, + [UNARY_NOT] = { true, INSTR_FMT_IX }, + [UNARY_INVERT] = { true, INSTR_FMT_IX }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC }, + [BINARY_SUBSCR] = { true, INSTR_FMT_IXC000 }, + [BINARY_SLICE] = { true, INSTR_FMT_IX }, + [STORE_SLICE] = { true, INSTR_FMT_IX }, + [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC000 }, + [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC000 }, + [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC000 }, + [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC000 }, + [LIST_APPEND] = { true, INSTR_FMT_IB }, + [SET_ADD] = { true, INSTR_FMT_IB }, + [STORE_SUBSCR] = { true, INSTR_FMT_IXC }, + [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC }, + [DELETE_SUBSCR] = { true, INSTR_FMT_IX }, + [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB }, + [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX }, + [RETURN_VALUE] = { true, INSTR_FMT_IX }, + [RETURN_CONST] = { true, INSTR_FMT_IB }, + [GET_AITER] = { true, INSTR_FMT_IX }, + [GET_ANEXT] = { true, INSTR_FMT_IX }, + [GET_AWAITABLE] = { true, INSTR_FMT_IB }, + [SEND] = { true, INSTR_FMT_IBC }, + [SEND_GEN] = { true, INSTR_FMT_IBC }, + [YIELD_VALUE] = { true, INSTR_FMT_IX }, + [POP_EXCEPT] = { true, INSTR_FMT_IX }, + [RERAISE] = { true, INSTR_FMT_IB }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX }, + [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX }, + [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX }, + [STORE_NAME] = { true, INSTR_FMT_IB }, + [DELETE_NAME] = { true, INSTR_FMT_IB }, + [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC }, + [UNPACK_EX] = { true, INSTR_FMT_IB }, + [STORE_ATTR] = { true, INSTR_FMT_IBC000 }, + [DELETE_ATTR] = { true, INSTR_FMT_IB }, + [STORE_GLOBAL] = { true, INSTR_FMT_IB }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB }, + [LOAD_NAME] = { true, INSTR_FMT_IB }, + [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000 }, + [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000 }, + [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000 }, + [DELETE_FAST] = { true, INSTR_FMT_IB }, + [MAKE_CELL] = { true, INSTR_FMT_IB }, + [DELETE_DEREF] = { true, INSTR_FMT_IB }, + [LOAD_CLASSDEREF] = { true, INSTR_FMT_IB }, + [LOAD_DEREF] = { true, INSTR_FMT_IB }, + [STORE_DEREF] = { true, INSTR_FMT_IB }, + [COPY_FREE_VARS] = { true, INSTR_FMT_IB }, + [BUILD_STRING] = { true, INSTR_FMT_IB }, + [BUILD_TUPLE] = { true, INSTR_FMT_IB }, + [BUILD_LIST] = { true, INSTR_FMT_IB }, + [LIST_EXTEND] = { true, INSTR_FMT_IB }, + [SET_UPDATE] = { true, INSTR_FMT_IB }, + [BUILD_SET] = { true, INSTR_FMT_IB }, + [BUILD_MAP] = { true, INSTR_FMT_IB }, + [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX }, + [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB }, + [DICT_UPDATE] = { true, INSTR_FMT_IB }, + [DICT_MERGE] = { true, INSTR_FMT_IB }, + [MAP_ADD] = { true, INSTR_FMT_IB }, + [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000 }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000 }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000 }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000 }, + [COMPARE_OP] = { true, INSTR_FMT_IBC }, + [COMPARE_AND_BRANCH] = { true, INSTR_FMT_IBC0 }, + [COMPARE_AND_BRANCH_FLOAT] = { true, INSTR_FMT_IBC0 }, + [COMPARE_AND_BRANCH_INT] = { true, INSTR_FMT_IBC0 }, + [COMPARE_AND_BRANCH_STR] = { true, INSTR_FMT_IBC0 }, + [IS_OP] = { true, INSTR_FMT_IB }, + [CONTAINS_OP] = { true, INSTR_FMT_IB }, + [CHECK_EG_MATCH] = { true, INSTR_FMT_IX }, + [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX }, + [IMPORT_NAME] = { true, INSTR_FMT_IB }, + [IMPORT_FROM] = { true, INSTR_FMT_IB }, + [JUMP_FORWARD] = { true, INSTR_FMT_IB }, + [JUMP_BACKWARD] = { true, INSTR_FMT_IB }, + [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IB }, + [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB }, + [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IB }, + [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IB }, + [JUMP_IF_FALSE_OR_POP] = { true, INSTR_FMT_IB }, + [JUMP_IF_TRUE_OR_POP] = { true, INSTR_FMT_IB }, + [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB }, + [GET_LEN] = { true, INSTR_FMT_IX }, + [MATCH_CLASS] = { true, INSTR_FMT_IB }, + [MATCH_MAPPING] = { true, INSTR_FMT_IX }, + [MATCH_SEQUENCE] = { true, INSTR_FMT_IX }, + [MATCH_KEYS] = { true, INSTR_FMT_IX }, + [GET_ITER] = { true, INSTR_FMT_IX }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX }, + [FOR_ITER] = { true, INSTR_FMT_IBC }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC }, + [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX }, + [BEFORE_WITH] = { true, INSTR_FMT_IX }, + [WITH_EXCEPT_START] = { true, INSTR_FMT_IX }, + [PUSH_EXC_INFO] = { true, INSTR_FMT_IX }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000 }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000 }, + [KW_NAMES] = { true, INSTR_FMT_IB }, + [CALL] = { true, INSTR_FMT_IBC000 }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC000 }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC000 }, + [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_TYPE_1] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_TUPLE_1] = { true, INSTR_FMT_IBC000 }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_BUILTIN_O] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_BUILTIN_FAST] = { true, INSTR_FMT_IBC000 }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_LEN] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_ISINSTANCE] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_LIST_APPEND] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC000 }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC000 }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IB }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX }, + [BUILD_SLICE] = { true, INSTR_FMT_IB }, + [FORMAT_VALUE] = { true, INSTR_FMT_IB }, + [COPY] = { true, INSTR_FMT_IB }, + [BINARY_OP] = { true, INSTR_FMT_IBC }, + [SWAP] = { true, INSTR_FMT_IB }, + [EXTENDED_ARG] = { true, INSTR_FMT_IB }, + [CACHE] = { true, INSTR_FMT_IX }, }; #endif diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index b16c927a4a2870..f1b655b8b0545b 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -229,7 +229,6 @@ class Instruction: # Parts of the underlying instruction definition inst: parser.InstDef - register: bool kind: typing.Literal["inst", "op", "legacy"] # Legacy means no (input -- output) name: str block: parser.Block @@ -246,17 +245,12 @@ class Instruction: unmoved_names: frozenset[str] instr_fmt: str - # Parallel to input_effects; set later - input_registers: list[str] = dataclasses.field(repr=False) - output_registers: list[str] = dataclasses.field(repr=False) - # Set later family: parser.Family | None = None predicted: bool = False def __init__(self, inst: parser.InstDef): self.inst = inst - self.register = inst.register self.kind = inst.kind self.name = inst.name self.block = inst.block @@ -278,15 +272,10 @@ def __init__(self, inst: parser.InstDef): else: break self.unmoved_names = frozenset(unmoved_names) - if self.register: - num_regs = len(self.input_effects) + len(self.output_effects) - num_dummies = (num_regs // 2) * 2 + 1 - num_regs - fmt = "I" + "B" * num_regs + "X" * num_dummies + if variable_used(inst, "oparg"): + fmt = "IB" else: - if variable_used(inst, "oparg"): - fmt = "IB" - else: - fmt = "IX" + fmt = "IX" cache = "C" for ce in self.cache_effects: for _ in range(ce.size): @@ -294,20 +283,6 @@ def __init__(self, inst: parser.InstDef): cache = "0" self.instr_fmt = fmt - def analyze_registers(self, a: "Analyzer") -> None: - regs = iter(("REG(oparg1)", "REG(oparg2)", "REG(oparg3)")) - try: - self.input_registers = [ - next(regs) for ieff in self.input_effects if ieff.name != UNUSED - ] - self.output_registers = [ - next(regs) for oeff in self.output_effects if oeff.name != UNUSED - ] - except StopIteration: # Running out of registers - a.error( - f"Instruction {self.name} has too many register effects", node=self.inst - ) - def write(self, out: Formatter) -> None: """Write one instruction, sans prologue and epilogue.""" # Write a static assertion that a family's cache size is correct @@ -319,25 +294,19 @@ def write(self, out: Formatter) -> None: f'{self.cache_offset}, "incorrect cache size");' ) - if not self.register: - # Write input stack effect variable declarations and initializations - ieffects = list(reversed(self.input_effects)) - for i, ieffect in enumerate(ieffects): - isize = string_effect_size( - list_effect_size([ieff for ieff in ieffects[: i + 1]]) - ) - if ieffect.size: - src = StackEffect(f"(stack_pointer - {maybe_parenthesize(isize)})", "PyObject **") - elif ieffect.cond: - src = StackEffect(f"({ieffect.cond}) ? stack_pointer[-{maybe_parenthesize(isize)}] : NULL", "") - else: - src = StackEffect(f"stack_pointer[-{maybe_parenthesize(isize)}]", "") - out.declare(ieffect, src) - else: - # Write input register variable declarations and initializations - for ieffect, reg in zip(self.input_effects, self.input_registers): - src = StackEffect(reg, "") - out.declare(ieffect, src) + # Write input stack effect variable declarations and initializations + ieffects = list(reversed(self.input_effects)) + for i, ieffect in enumerate(ieffects): + isize = string_effect_size( + list_effect_size([ieff for ieff in ieffects[: i + 1]]) + ) + if ieffect.size: + src = StackEffect(f"(stack_pointer - {maybe_parenthesize(isize)})", "PyObject **") + elif ieffect.cond: + src = StackEffect(f"({ieffect.cond}) ? stack_pointer[-{maybe_parenthesize(isize)}] : NULL", "") + else: + src = StackEffect(f"stack_pointer[-{maybe_parenthesize(isize)}]", "") + out.declare(ieffect, src) # Write output stack effect variable declarations isize = string_effect_size(list_effect_size(self.input_effects)) @@ -367,32 +336,26 @@ def write(self, out: Formatter) -> None: if self.always_exits: return - if not self.register: - # Write net stack growth/shrinkage - out.stack_adjust( - 0, - [ieff for ieff in self.input_effects], - [oeff for oeff in self.output_effects], - ) + # Write net stack growth/shrinkage + out.stack_adjust( + 0, + [ieff for ieff in self.input_effects], + [oeff for oeff in self.output_effects], + ) - # Write output stack effect assignments - oeffects = list(reversed(self.output_effects)) - for i, oeffect in enumerate(oeffects): - if oeffect.name in self.unmoved_names: - continue - osize = string_effect_size( - list_effect_size([oeff for oeff in oeffects[: i + 1]]) - ) - if oeffect.size: - dst = StackEffect(f"stack_pointer - {maybe_parenthesize(osize)}", "PyObject **") - else: - dst = StackEffect(f"stack_pointer[-{maybe_parenthesize(osize)}]", "") - out.assign(dst, oeffect) - else: - # Write output register assignments - for oeffect, reg in zip(self.output_effects, self.output_registers): - dst = StackEffect(reg, "") - out.assign(dst, oeffect) + # Write output stack effect assignments + oeffects = list(reversed(self.output_effects)) + for i, oeffect in enumerate(oeffects): + if oeffect.name in self.unmoved_names: + continue + osize = string_effect_size( + list_effect_size([oeff for oeff in oeffects[: i + 1]]) + ) + if oeffect.size: + dst = StackEffect(f"stack_pointer - {maybe_parenthesize(osize)}", "PyObject **") + else: + dst = StackEffect(f"stack_pointer[-{maybe_parenthesize(osize)}]", "") + out.assign(dst, oeffect) # Write cache effect if self.cache_offset: @@ -438,19 +401,17 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None # ERROR_IF() must pop the inputs from the stack. # The code block is responsible for DECREF()ing them. # NOTE: If the label doesn't exist, just add it to ceval.c. - if not self.register: - # Don't pop common input/output effects at the bottom! - # These aren't DECREF'ed so they can stay. - ieffs = list(self.input_effects) - oeffs = list(self.output_effects) - while ieffs and oeffs and ieffs[0] == oeffs[0]: - ieffs.pop(0) - oeffs.pop(0) - ninputs, symbolic = list_effect_size(ieffs) - if ninputs: - label = f"pop_{ninputs}_{label}" - else: - symbolic = "" + + # Don't pop common input/output effects at the bottom! + # These aren't DECREF'ed so they can stay. + ieffs = list(self.input_effects) + oeffs = list(self.output_effects) + while ieffs and oeffs and ieffs[0] == oeffs[0]: + ieffs.pop(0) + oeffs.pop(0) + ninputs, symbolic = list_effect_size(ieffs) + if ninputs: + label = f"pop_{ninputs}_{label}" if symbolic: out.write_raw( f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n" @@ -458,21 +419,20 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None else: out.write_raw(f"{space}if ({cond}) goto {label};\n") elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): - if not self.register: - out.reset_lineno() - space = extra + m.group(1) - for ieff in self.input_effects: - if ieff.name in names_to_skip: - continue - if ieff.size: - out.write_raw( - f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n" - ) - out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n") - out.write_raw(f"{space}}}\n") - else: - decref = "XDECREF" if ieff.cond else "DECREF" - out.write_raw(f"{space}Py_{decref}({ieff.name});\n") + out.reset_lineno() + space = extra + m.group(1) + for ieff in self.input_effects: + if ieff.name in names_to_skip: + continue + if ieff.size: + out.write_raw( + f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n" + ) + out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n") + out.write_raw(f"{space}}}\n") + else: + decref = "XDECREF" if ieff.cond else "DECREF" + out.write_raw(f"{space}Py_{decref}({ieff.name});\n") else: out.write_raw(extra + line) out.reset_lineno() @@ -672,7 +632,6 @@ def analyze(self) -> None: Raises SystemExit if there is an error. """ self.find_predictions() - self.analyze_register_instrs() self.analyze_supers_and_macros() self.map_families() self.check_families() @@ -782,11 +741,6 @@ def effect_counts(self, name: str) -> tuple[int, int, int]: assert False, f"Unknown instruction {name!r}" return cache, input, output - def analyze_register_instrs(self) -> None: - for instr in self.instrs.values(): - if instr.register: - instr.analyze_registers(self) - def analyze_supers_and_macros(self) -> None: """Analyze each super- and macro instruction.""" self.super_instrs = {} @@ -816,7 +770,7 @@ def analyze_macro(self, macro: parser.Macro) -> MacroInstruction: stack, initial_sp = self.stack_analysis(components) sp = initial_sp parts: list[Component | parser.CacheEffect] = [] - format = "IB" # Macros don't support register instructions yet + format = "IB" cache = "C" for component in components: match component: @@ -1034,13 +988,9 @@ def write_metadata(self) -> None: self.write_stack_effect_functions() # Write type definitions - self.out.emit("enum Direction { DIR_NONE, DIR_READ, DIR_WRITE };") self.out.emit(f"enum InstructionFormat {{ {', '.join(format_enums)} }};") self.out.emit("struct opcode_metadata {") with self.out.indent(): - self.out.emit("enum Direction dir_op1;") - self.out.emit("enum Direction dir_op2;") - self.out.emit("enum Direction dir_op3;") self.out.emit("bool valid_entry;") self.out.emit("enum InstructionFormat instr_format;") self.out.emit("};") @@ -1073,32 +1023,20 @@ def write_metadata(self) -> None: def write_metadata_for_inst(self, instr: Instruction) -> None: """Write metadata for a single instruction.""" - dir_op1 = dir_op2 = dir_op3 = "DIR_NONE" - if instr.kind == "legacy": - assert not instr.register - else: - if instr.register: - directions: list[str] = [] - directions.extend("DIR_READ" for _ in instr.input_effects) - directions.extend("DIR_WRITE" for _ in instr.output_effects) - directions.extend("DIR_NONE" for _ in range(3)) - dir_op1, dir_op2, dir_op3 = directions[:3] self.out.emit( - f" [{instr.name}] = {{ {dir_op1}, {dir_op2}, {dir_op3}, true, {INSTR_FMT_PREFIX}{instr.instr_fmt} }}," + f" [{instr.name}] = {{ true, {INSTR_FMT_PREFIX}{instr.instr_fmt} }}," ) def write_metadata_for_super(self, sup: SuperInstruction) -> None: """Write metadata for a super-instruction.""" - dir_op1 = dir_op2 = dir_op3 = "DIR_NONE" self.out.emit( - f" [{sup.name}] = {{ {dir_op1}, {dir_op2}, {dir_op3}, true, {INSTR_FMT_PREFIX}{sup.instr_fmt} }}," + f" [{sup.name}] = {{ true, {INSTR_FMT_PREFIX}{sup.instr_fmt} }}," ) def write_metadata_for_macro(self, mac: MacroInstruction) -> None: """Write metadata for a macro-instruction.""" - dir_op1 = dir_op2 = dir_op3 = "DIR_NONE" self.out.emit( - f" [{mac.name}] = {{ {dir_op1}, {dir_op2}, {dir_op3}, true, {INSTR_FMT_PREFIX}{mac.instr_fmt} }}," + f" [{mac.name}] = {{ true, {INSTR_FMT_PREFIX}{mac.instr_fmt} }}," ) def write_instructions(self) -> None: From 2a03ed034edc6bb2db6fb972b2efe08db5145a28 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Mar 2023 18:43:54 -0600 Subject: [PATCH 16/34] gh-102660: Fix Refleaks in import.c (#102744) gh-102661 introduced some leaks. This fixes them. https://github.com/python/cpython/issues/102660 --- Python/bltinmodule.c | 3 --- Python/import.c | 50 +++++++++++++++++++++++++------------------- Python/sysmodule.c | 3 --- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index a8a34620b9bcdf..12ca0ba6c4873c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3098,9 +3098,6 @@ _PyBuiltin_Init(PyInterpreterState *interp) } Py_DECREF(debug); - /* m_copy of Py_None means it is copied some other way. */ - builtinsmodule.m_base.m_copy = Py_NewRef(Py_None); - return mod; #undef ADD_TO_ALL #undef SETBUILTIN diff --git a/Python/import.c b/Python/import.c index 612fee243bd9e7..9f80c6d8dd49a8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -978,14 +978,29 @@ _PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) return 0; } -static inline int -match_mod_name(PyObject *actual, const char *expected) +static PyObject * +get_core_module_dict(PyInterpreterState *interp, + PyObject *name, PyObject *filename) { - if (PyUnicode_CompareWithASCIIString(actual, expected) == 0) { - return 1; + /* Only builtin modules are core. */ + if (filename == name) { + assert(!PyErr_Occurred()); + if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { + return interp->sysdict_copy; + } + assert(!PyErr_Occurred()); + if (PyUnicode_CompareWithASCIIString(name, "builtins") == 0) { + return interp->builtins_copy; + } + assert(!PyErr_Occurred()); } - assert(!PyErr_Occurred()); - return 0; + return NULL; +} + +static inline int +is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *filename) +{ + return get_core_module_dict(interp, name, filename) != NULL; } static int @@ -1009,10 +1024,8 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) // bpo-44050: Extensions and def->m_base.m_copy can be updated // when the extension module doesn't support sub-interpreters. - // XXX Why special-case the main interpreter? - if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { - /* m_copy of Py_None means it is copied some other way. */ - if (def->m_size == -1 && def->m_base.m_copy != Py_None) { + if (def->m_size == -1) { + if (!is_core_module(tstate->interp, name, filename)) { if (def->m_base.m_copy) { /* Somebody already imported the module, likely under a different name. @@ -1028,7 +1041,10 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) return -1; } } + } + // XXX Why special-case the main interpreter? + if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { if (_extensions_cache_set(filename, name, def) < 0) { return -1; } @@ -1069,21 +1085,11 @@ import_find_extension(PyThreadState *tstate, PyObject *name, PyObject *m_copy = def->m_base.m_copy; /* Module does not support repeated initialization */ if (m_copy == NULL) { - return NULL; - } - else if (m_copy == Py_None) { - if (match_mod_name(name, "sys")) { - m_copy = tstate->interp->sysdict_copy; - } - else if (match_mod_name(name, "builtins")) { - m_copy = tstate->interp->builtins_copy; - } - else { - _PyErr_SetString(tstate, PyExc_ImportError, "missing m_copy"); + m_copy = get_core_module_dict(tstate->interp, name, filename); + if (m_copy == NULL) { return NULL; } } - /* m_copy of Py_None means it is copied some other way. */ mod = import_add_module(tstate, name); if (mod == NULL) { return NULL; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index fc0550266bf1af..d282104dcad414 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3425,9 +3425,6 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) return _PyStatus_ERR("failed to create a module object"); } - /* m_copy of Py_None means it is copied some other way. */ - sysmodule.m_base.m_copy = Py_NewRef(Py_None); - PyObject *sysdict = PyModule_GetDict(sysmod); if (sysdict == NULL) { goto error; From 1c9f3391b916939e5ad18213e553f8d6bfbec25e Mon Sep 17 00:00:00 2001 From: Jamoo721 <81095953+Jamoo721@users.noreply.github.com> Date: Thu, 16 Mar 2023 13:52:11 +1100 Subject: [PATCH 17/34] gh-102690: Use Edge as fallback in webbrowser instead of IE (#102691) --- Lib/webbrowser.py | 12 ++++++++---- .../2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 44974d433b4696..a56ff33dbbdc69 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -542,11 +542,15 @@ def register_standard_browsers(): # First try to use the default Windows browser register("windows-default", WindowsDefault) - # Detect some common Windows browsers, fallback to IE - iexplore = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"), - "Internet Explorer\\IEXPLORE.EXE") + # Detect some common Windows browsers, fallback to Microsoft Edge + # location in 64-bit Windows + edge64 = os.path.join(os.environ.get("PROGRAMFILES(x86)", "C:\\Program Files (x86)"), + "Microsoft\\Edge\\Application\\msedge.exe") + # location in 32-bit Windows + edge32 = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"), + "Microsoft\\Edge\\Application\\msedge.exe") for browser in ("firefox", "firebird", "seamonkey", "mozilla", - "netscape", "opera", iexplore): + "opera", edge64, edge32): if shutil.which(browser): register(browser, None, BackgroundBrowser(browser)) else: diff --git a/Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst b/Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst new file mode 100644 index 00000000000000..5669ebbb442c24 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst @@ -0,0 +1 @@ +Update :mod:`webbrowser` to fall back to Microsoft Edge instead of Internet Explorer. From a44553ea9f7745a1119148082edb1fb0372ac0e2 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 16 Mar 2023 09:20:43 +0530 Subject: [PATCH 18/34] GH-100112: avoid using iterable coroutines in asyncio internally (#100128) --- Lib/asyncio/tasks.py | 22 +++++-------------- Lib/test/test_asyncio/test_tasks.py | 15 +++++++++++++ ...-03-10-13-51-21.gh-issue-100112.VHh4mw.rst | 1 + 3 files changed, 22 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 1c20754b839b69..c90d32c97add78 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -25,7 +25,6 @@ from . import exceptions from . import futures from . import timeouts -from .coroutines import _is_coroutine # Helper to generate new task names # This uses itertools.count() instead of a "+= 1" operation because the latter @@ -635,11 +634,14 @@ def ensure_future(coro_or_future, *, loop=None): raise ValueError('The future belongs to a different loop than ' 'the one specified as the loop argument') return coro_or_future - called_wrap_awaitable = False + should_close = True if not coroutines.iscoroutine(coro_or_future): if inspect.isawaitable(coro_or_future): + async def _wrap_awaitable(awaitable): + return await awaitable + coro_or_future = _wrap_awaitable(coro_or_future) - called_wrap_awaitable = True + should_close = False else: raise TypeError('An asyncio.Future, a coroutine or an awaitable ' 'is required') @@ -649,23 +651,11 @@ def ensure_future(coro_or_future, *, loop=None): try: return loop.create_task(coro_or_future) except RuntimeError: - if not called_wrap_awaitable: + if should_close: coro_or_future.close() raise -@types.coroutine -def _wrap_awaitable(awaitable): - """Helper for asyncio.ensure_future(). - - Wraps awaitable (an object with __await__) into a coroutine - that will later be wrapped in a Task by ensure_future(). - """ - return (yield from awaitable.__await__()) - -_wrap_awaitable._is_coroutine = _is_coroutine - - class _GatheringFuture(futures.Future): """Helper for gather(). diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index e533d5273e9f38..5b935b526541a1 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -8,6 +8,7 @@ import re import sys import traceback +import types import unittest from unittest import mock from types import GenericAlias @@ -274,6 +275,20 @@ async def coro(): loop.run_until_complete(fut) self.assertEqual(fut.result(), 'ok') + def test_ensure_future_task_awaitable(self): + class Aw: + def __await__(self): + return asyncio.sleep(0, result='ok').__await__() + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + task = asyncio.ensure_future(Aw(), loop=loop) + loop.run_until_complete(task) + self.assertTrue(task.done()) + self.assertEqual(task.result(), 'ok') + self.assertIsInstance(task.get_coro(), types.CoroutineType) + loop.close() + def test_ensure_future_neither(self): with self.assertRaises(TypeError): asyncio.ensure_future('ok') diff --git a/Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst b/Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst new file mode 100644 index 00000000000000..eff77c40e30c48 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst @@ -0,0 +1 @@ +:meth:`asyncio.Task.get_coro` now always returns a coroutine when wrapping an awaitable object. Patch by Kumar Aditya. From 2dc94634b50f0e5e207787e5ac1d56c68b22c3ae Mon Sep 17 00:00:00 2001 From: yonatanp Date: Thu, 16 Mar 2023 00:44:52 -0400 Subject: [PATCH 19/34] gh-94440: Fix issue of ProcessPoolExecutor shutdown hanging (#94468) Fix an issue of concurrent.futures ProcessPoolExecutor shutdown hanging. Co-authored-by: Alex Waygood --- Lib/concurrent/futures/process.py | 5 ++++ Lib/test/test_concurrent_futures.py | 28 +++++++++++++++++++ Misc/ACKS | 1 + ...2-06-30-21-28-41.gh-issue-94440.LtgX0d.rst | 2 ++ 4 files changed, 36 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 3a8637b6fa1b47..816edab99f63e3 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -366,6 +366,11 @@ def run(self): if self.is_shutting_down(): self.flag_executor_shutting_down() + # When only canceled futures remain in pending_work_items, our + # next call to wait_result_broken_or_wakeup would hang forever. + # This makes sure we have some running futures or none at all. + self.add_call_item_to_queue() + # Since no new work items can be added, it is safe to shutdown # this thread if there are no pending work items. if not self.pending_work_items: diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index b3520ae3994e03..a20cb844a293c9 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -14,6 +14,7 @@ from logging.handlers import QueueHandler import os import queue +import signal import sys import threading import time @@ -397,6 +398,33 @@ def test_hang_gh83386(self): self.assertFalse(err) self.assertEqual(out.strip(), b"apple") + def test_hang_gh94440(self): + """shutdown(wait=True) doesn't hang when a future was submitted and + quickly canceled right before shutdown. + + See https://github.com/python/cpython/issues/94440. + """ + if not hasattr(signal, 'alarm'): + raise unittest.SkipTest( + "Tested platform does not support the alarm signal") + + def timeout(_signum, _frame): + raise RuntimeError("timed out waiting for shutdown") + + kwargs = {} + if getattr(self, 'ctx', None): + kwargs['mp_context'] = self.get_context() + executor = self.executor_type(max_workers=1, **kwargs) + executor.submit(int).result() + old_handler = signal.signal(signal.SIGALRM, timeout) + try: + signal.alarm(5) + executor.submit(int).cancel() + executor.shutdown(wait=True) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, BaseTestCase): def test_threads_terminate(self): diff --git a/Misc/ACKS b/Misc/ACKS index 7bbde3af99782b..8cf5166a2bb1f4 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1385,6 +1385,7 @@ Thomas Perl Mathieu Perreault Mark Perrego Trevor Perrin +Yonatan Perry Gabriel de Perthuis Tim Peters Benjamin Peterson diff --git a/Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst b/Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst new file mode 100644 index 00000000000000..3eee82e59dfafb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst @@ -0,0 +1,2 @@ +Fix a :mod:`concurrent.futures.process` bug where ``ProcessPoolExecutor`` shutdown +could hang after a future has been quickly submitted and canceled. From 51d693c58454a2c525094a7c74ebac86859353fd Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 16 Mar 2023 10:16:01 +0000 Subject: [PATCH 20/34] gh-102594: PyErr_SetObject adds note to exception raised on normalization error (#102675) --- Include/cpython/pyerrors.h | 4 ++ Lib/test/test_capi/test_exceptions.py | 20 ++++++++++ ...-03-14-00-11-46.gh-issue-102594.BjU-m2.rst | 1 + Modules/_testcapi/exceptions.c | 21 ++++++++++ Objects/exceptions.c | 15 +++++++ Python/errors.c | 40 ++++++++++++++++--- 6 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 0d9cc9922f7368..d0300f6ee56a25 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -112,6 +112,10 @@ PyAPI_FUNC(PyObject *) _PyErr_FormatFromCause( /* In exceptions.c */ +PyAPI_FUNC(int) _PyException_AddNote( + PyObject *exc, + PyObject *note); + /* Helper that attempts to replace the current exception with one of the * same type but with a prefix added to the exception text. The resulting * exception description looks like: diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index 55f131699a2567..b1c1a61e20685e 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -169,5 +169,25 @@ class Broken(Exception, metaclass=Meta): with self.assertRaises(ZeroDivisionError) as e: _testcapi.exc_set_object(Broken, Broken()) + def test_set_object_and_fetch(self): + class Broken(Exception): + def __init__(self, *arg): + raise ValueError("Broken __init__") + + exc = _testcapi.exc_set_object_fetch(Broken, 'abcd') + self.assertIsInstance(exc, ValueError) + self.assertEqual(exc.__notes__[0], + "Normalization failed: type=Broken args='abcd'") + + class BadArg: + def __repr__(self): + raise TypeError('Broken arg type') + + exc = _testcapi.exc_set_object_fetch(Broken, BadArg()) + self.assertIsInstance(exc, ValueError) + self.assertEqual(exc.__notes__[0], + 'Normalization failed: type=Broken args=') + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst new file mode 100644 index 00000000000000..0b95b5ec98e811 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst @@ -0,0 +1 @@ +Add note to exception raised in ``PyErr_SetObject`` when normalization fails. diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index a0575213987ffc..c64b823663c32f 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -92,6 +92,26 @@ exc_set_object(PyObject *self, PyObject *args) return NULL; } +static PyObject * +exc_set_object_fetch(PyObject *self, PyObject *args) +{ + PyObject *exc; + PyObject *obj; + PyObject *type; + PyObject *value; + PyObject *tb; + + if (!PyArg_ParseTuple(args, "OO:exc_set_object", &exc, &obj)) { + return NULL; + } + + PyErr_SetObject(exc, obj); + PyErr_Fetch(&type, &value, &tb); + Py_XDECREF(type); + Py_XDECREF(tb); + return value; +} + static PyObject * raise_exception(PyObject *self, PyObject *args) { @@ -262,6 +282,7 @@ static PyMethodDef test_methods[] = { {"make_exception_with_doc", _PyCFunction_CAST(make_exception_with_doc), METH_VARARGS | METH_KEYWORDS}, {"exc_set_object", exc_set_object, METH_VARARGS}, + {"exc_set_object_fetch", exc_set_object_fetch, METH_VARARGS}, {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", raise_memoryerror, METH_NOARGS}, {"set_exc_info", test_set_exc_info, METH_VARARGS}, diff --git a/Objects/exceptions.c b/Objects/exceptions.c index c6fb6a3f19b2d0..d69f7400ca6042 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3749,6 +3749,21 @@ _PyExc_Fini(PyInterpreterState *interp) _PyExc_FiniTypes(interp); } +int +_PyException_AddNote(PyObject *exc, PyObject *note) +{ + if (!PyExceptionInstance_Check(exc)) { + PyErr_Format(PyExc_TypeError, + "exc must be an exception, not '%s'", + Py_TYPE(exc)->tp_name); + return -1; + } + PyObject *r = BaseException_add_note(exc, note); + int res = r == NULL ? -1 : 0; + Py_XDECREF(r); + return res; +} + /* Helper to do the equivalent of "raise X from Y" in C, but always using * the current exception rather than passing one in. * diff --git a/Python/errors.c b/Python/errors.c index bbf6d397ce8097..bdcbac317eb9ee 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -135,6 +135,28 @@ _PyErr_GetTopmostException(PyThreadState *tstate) return exc_info; } +static PyObject * +get_normalization_failure_note(PyThreadState *tstate, PyObject *exception, PyObject *value) +{ + PyObject *args = PyObject_Repr(value); + if (args == NULL) { + _PyErr_Clear(tstate); + args = PyUnicode_FromFormat(""); + } + PyObject *note; + const char *tpname = ((PyTypeObject*)exception)->tp_name; + if (args == NULL) { + _PyErr_Clear(tstate); + note = PyUnicode_FromFormat("Normalization failed: type=%s", tpname); + } + else { + note = PyUnicode_FromFormat("Normalization failed: type=%s args=%S", + tpname, args); + Py_DECREF(args); + } + return note; +} + void _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) { @@ -160,19 +182,27 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) Py_XINCREF(value); if (!is_subclass) { /* We must normalize the value right now */ - PyObject *fixed_value; /* Issue #23571: functions must not be called with an exception set */ _PyErr_Clear(tstate); - fixed_value = _PyErr_CreateException(exception, value); - Py_XDECREF(value); + PyObject *fixed_value = _PyErr_CreateException(exception, value); if (fixed_value == NULL) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + assert(PyExceptionInstance_Check(exc)); + + PyObject *note = get_normalization_failure_note(tstate, exception, value); + Py_XDECREF(value); + if (note != NULL) { + /* ignore errors in _PyException_AddNote - they will be overwritten below */ + _PyException_AddNote(exc, note); + Py_DECREF(note); + } + _PyErr_SetRaisedException(tstate, exc); return; } - - value = fixed_value; + Py_XSETREF(value, fixed_value); } exc_value = _PyErr_GetTopmostException(tstate)->exc_value; From a297d59609038ccfc3bdf6f350e8401f07b0a931 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 16 Mar 2023 16:05:38 +0300 Subject: [PATCH 21/34] Add comments to `{typing,_collections_abc}._type_repr` about each other (#102752) Remove `if` condition in `_collections_abc._type_repr` that's no longer needed, bringing it in sync with `typing._type_repr`. --- Lib/_collections_abc.py | 3 +-- Lib/typing.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c62233b81a5c95..f86b91a5e6fb0d 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -517,9 +517,8 @@ def _type_repr(obj): Copied from :mod:`typing` since collections.abc shouldn't depend on that module. + (Keep this roughly in sync with the typing version.) """ - if isinstance(obj, GenericAlias): - return repr(obj) if isinstance(obj, type): if obj.__module__ == 'builtins': return obj.__qualname__ diff --git a/Lib/typing.py b/Lib/typing.py index ab334395676159..3ee9679e50c0c4 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -230,6 +230,9 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ + # When changing this function, don't forget about + # `_collections_abc._type_repr`, which does the same thing + # and must be consistent with this one. if isinstance(obj, type): if obj.__module__ == 'builtins': return obj.__qualname__ From b0ec6253c9cf1b22b87cd99f317dbaeb31263b20 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 16 Mar 2023 09:32:18 -0500 Subject: [PATCH 22/34] GH-102653: Make recipe docstring show the correct distribution (#102742) --- Doc/library/random.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 4522f5a8d26b9d..098684d7270ffa 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -610,7 +610,8 @@ from the combinatoric iterators in the :mod:`itertools` module: return tuple(pool[i] for i in indices) def random_combination_with_replacement(iterable, r): - "Random selection from itertools.combinations_with_replacement(iterable, r)" + "Choose r elements with replacement. Order the result to match the iterable." + # Result will be in set(itertools.combinations_with_replacement(iterable, r)). pool = tuple(iterable) n = len(pool) indices = sorted(random.choices(range(n), k=r)) From fbe82fdd777cead5d6e08ac0d1da19738c19bd81 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 16 Mar 2023 17:47:30 +0300 Subject: [PATCH 23/34] gh-102721: Improve coverage of `_collections_abc._CallableGenericAlias` (#102722) Co-authored-by: Alex Waygood --- Lib/_collections_abc.py | 7 ------- Lib/test/test_typing.py | 39 ++++++++++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index f86b91a5e6fb0d..9d7724c33474cc 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -481,15 +481,8 @@ def __getitem__(self, item): # rather than the default types.GenericAlias object. Most of the # code is copied from typing's _GenericAlias and the builtin # types.GenericAlias. - if not isinstance(item, tuple): item = (item,) - # A special case in PEP 612 where if X = Callable[P, int], - # then X[int, str] == X[[int, str]]. - if (len(self.__parameters__) == 1 - and _is_param_expr(self.__parameters__[0]) - and item and not _is_param_expr(item[0])): - item = (item,) new_args = super().__getitem__(item).__args__ diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 89c725cda54f12..c9f55de95c548f 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1921,14 +1921,29 @@ def test_weakref(self): self.assertEqual(weakref.ref(alias)(), alias) def test_pickle(self): + global T_pickle, P_pickle, TS_pickle # needed for pickling Callable = self.Callable - alias = Callable[[int, str], float] - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - s = pickle.dumps(alias, proto) - loaded = pickle.loads(s) - self.assertEqual(alias.__origin__, loaded.__origin__) - self.assertEqual(alias.__args__, loaded.__args__) - self.assertEqual(alias.__parameters__, loaded.__parameters__) + T_pickle = TypeVar('T_pickle') + P_pickle = ParamSpec('P_pickle') + TS_pickle = TypeVarTuple('TS_pickle') + + samples = [ + Callable[[int, str], float], + Callable[P_pickle, int], + Callable[P_pickle, T_pickle], + Callable[Concatenate[int, P_pickle], int], + Callable[Concatenate[*TS_pickle, P_pickle], int], + ] + for alias in samples: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(alias=alias, proto=proto): + s = pickle.dumps(alias, proto) + loaded = pickle.loads(s) + self.assertEqual(alias.__origin__, loaded.__origin__) + self.assertEqual(alias.__args__, loaded.__args__) + self.assertEqual(alias.__parameters__, loaded.__parameters__) + + del T_pickle, P_pickle, TS_pickle # cleaning up global state def test_var_substitution(self): Callable = self.Callable @@ -1954,6 +1969,16 @@ def test_var_substitution(self): self.assertEqual(C5[int, str, float], Callable[[typing.List[int], tuple[str, int], float], int]) + def test_type_subst_error(self): + Callable = self.Callable + P = ParamSpec('P') + T = TypeVar('T') + + pat = "Expected a list of types, an ellipsis, ParamSpec, or Concatenate." + + with self.assertRaisesRegex(TypeError, pat): + Callable[P, T][0, int] + def test_type_erasure(self): Callable = self.Callable class C1(Callable): From adaed17341e649fabb0f522e9b4f5962fcdf1d48 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 16 Mar 2023 20:28:10 +0530 Subject: [PATCH 24/34] GH-102748: remove legacy support for generator based coroutines from `asyncio.iscoroutine` (#102749) Co-authored-by: Alex Waygood --- Doc/whatsnew/3.12.rst | 4 ++++ Lib/asyncio/coroutines.py | 3 +-- Lib/test/test_asyncio/test_pep492.py | 6 ++++++ .../Library/2023-03-16-08-17-29.gh-issue-102748.WNACpI.rst | 3 +++ 4 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-16-08-17-29.gh-issue-102748.WNACpI.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 217ffec1ee1007..b9c668543e1b73 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -237,6 +237,10 @@ asyncio * Add C implementation of :func:`asyncio.current_task` for 4x-6x speedup. (Contributed by Itamar Ostricher and Pranav Thulasiram Bhat in :gh:`100344`.) +* :func:`asyncio.iscoroutine` now returns ``False`` for generators as + :mod:`asyncio` does not support legacy generator-based coroutines. + (Contributed by Kumar Aditya in :gh:`102748`.) + inspect ------- diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index 7fda0e449d500a..ab4f30eb51ba2c 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -25,8 +25,7 @@ def iscoroutinefunction(func): # Prioritize native coroutine check to speed-up # asyncio.iscoroutine. -_COROUTINE_TYPES = (types.CoroutineType, types.GeneratorType, - collections.abc.Coroutine) +_COROUTINE_TYPES = (types.CoroutineType, collections.abc.Coroutine) _iscoroutine_typecache = set() diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index f833f788dcb98f..dc25a46985e349 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -119,6 +119,12 @@ async def foo(): pass self.assertTrue(asyncio.iscoroutine(FakeCoro())) + def test_iscoroutine_generator(self): + def foo(): yield + + self.assertFalse(asyncio.iscoroutine(foo())) + + def test_iscoroutinefunction(self): async def foo(): pass self.assertTrue(asyncio.iscoroutinefunction(foo)) diff --git a/Misc/NEWS.d/next/Library/2023-03-16-08-17-29.gh-issue-102748.WNACpI.rst b/Misc/NEWS.d/next/Library/2023-03-16-08-17-29.gh-issue-102748.WNACpI.rst new file mode 100644 index 00000000000000..b1dc67f38fe85d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-16-08-17-29.gh-issue-102748.WNACpI.rst @@ -0,0 +1,3 @@ +:func:`asyncio.iscoroutine` now returns ``False`` for generators as +:mod:`asyncio` does not support legacy generator-based coroutines. +Patch by Kumar Aditya. From 84e20c689a8b3b6cebfd50d044c62af5d0e7dec1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 16 Mar 2023 09:26:42 -0600 Subject: [PATCH 25/34] gh-102737: Un-ignore ceval.c in the CI globals check (gh-102745) The tool now allows user-added #LINE preprocessor directives. https://github.com/python/cpython/issues/102737 --- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 10 +++++++--- Tools/c-analyzer/cpython/_parser.py | 4 ---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 24c1b0e9b9d48c..c680f351f22416 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -153,9 +153,13 @@ def _iter_top_include_lines(lines, topfile, cwd, # XXX How can a file return to line 1? #assert lno > 1, (line, lno) else: - # It's the next line from the file. - assert included == files[-1], (line, files) - assert lno > 1, (line, lno) + if included == files[-1]: + # It's the next line from the file. + assert lno > 1, (line, lno) + else: + # We ran into a user-added #LINE directive, + # which we promptly ignore. + pass elif not files: raise NotImplementedError((line,)) elif filter_reqfile(files[-1]): diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index a2911e030ffee1..acf30e2c4020b3 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -96,10 +96,6 @@ def clean_lines(text): # has gone wrong where # we handle "maybe inline actual" # in Tools/c-analyzer/c_parser/parser/_global.py. Modules/expat/xmlparse.c - -# The parser doesn't like the #line directives -# that originate from generated_cases.c.h -Python/ceval.c ''') INCL_DIRS = clean_lines(''' From 61d6c110d6019a7ba79c76cc50aeaa85a1b37c08 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 16 Mar 2023 16:21:49 +0000 Subject: [PATCH 26/34] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives (#102743) --- Python/pythonrun.c | 79 ++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 44 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 34a44dd92847ba..5381b105a39fed 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -698,30 +698,30 @@ _Py_HandleSystemExit(int *exitcode_p) return 0; } - PyObject *exception, *value, *tb; - PyErr_Fetch(&exception, &value, &tb); - fflush(stdout); int exitcode = 0; - if (value == NULL || value == Py_None) { + + PyObject *exc = PyErr_GetRaisedException(); + if (exc == NULL) { goto done; } + assert(PyExceptionInstance_Check(exc)); - if (PyExceptionInstance_Check(value)) { - /* The error code should be in the `code' attribute. */ - PyObject *code = PyObject_GetAttr(value, &_Py_ID(code)); - if (code) { - Py_SETREF(value, code); - if (value == Py_None) - goto done; + /* The error code should be in the `code' attribute. */ + PyObject *code = PyObject_GetAttr(exc, &_Py_ID(code)); + if (code) { + Py_SETREF(exc, code); + if (exc == Py_None) { + goto done; } - /* If we failed to dig out the 'code' attribute, - just let the else clause below print the error. */ } + /* If we failed to dig out the 'code' attribute, + * just let the else clause below print the error. + */ - if (PyLong_Check(value)) { - exitcode = (int)PyLong_AsLong(value); + if (PyLong_Check(exc)) { + exitcode = (int)PyLong_AsLong(exc); } else { PyThreadState *tstate = _PyThreadState_GET(); @@ -732,20 +732,17 @@ _Py_HandleSystemExit(int *exitcode_p) */ PyErr_Clear(); if (sys_stderr != NULL && sys_stderr != Py_None) { - PyFile_WriteObject(value, sys_stderr, Py_PRINT_RAW); + PyFile_WriteObject(exc, sys_stderr, Py_PRINT_RAW); } else { - PyObject_Print(value, stderr, Py_PRINT_RAW); + PyObject_Print(exc, stderr, Py_PRINT_RAW); fflush(stderr); } PySys_WriteStderr("\n"); exitcode = 1; } - done: - /* Cleanup the exception */ - Py_CLEAR(exception); - Py_CLEAR(value); - Py_CLEAR(tb); +done: + Py_CLEAR(exc); *exitcode_p = exitcode; return 1; } @@ -1641,35 +1638,29 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, } - static void -flush_io(void) +flush_io_stream(PyThreadState *tstate, PyObject *name) { - PyObject *f, *r; - PyObject *type, *value, *traceback; - - /* Save the current exception */ - PyErr_Fetch(&type, &value, &traceback); - - PyThreadState *tstate = _PyThreadState_GET(); - f = _PySys_GetAttr(tstate, &_Py_ID(stderr)); - if (f != NULL) { - r = _PyObject_CallMethodNoArgs(f, &_Py_ID(flush)); - if (r) - Py_DECREF(r); - else - PyErr_Clear(); - } - f = _PySys_GetAttr(tstate, &_Py_ID(stdout)); + PyObject *f = _PySys_GetAttr(tstate, name); if (f != NULL) { - r = _PyObject_CallMethodNoArgs(f, &_Py_ID(flush)); - if (r) + PyObject *r = _PyObject_CallMethodNoArgs(f, &_Py_ID(flush)); + if (r) { Py_DECREF(r); - else + } + else { PyErr_Clear(); + } } +} - PyErr_Restore(type, value, traceback); +static void +flush_io(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *exc = _PyErr_GetRaisedException(tstate); + flush_io_stream(tstate, &_Py_ID(stderr)); + flush_io_stream(tstate, &_Py_ID(stdout)); + _PyErr_SetRaisedException(tstate, exc); } static PyObject * From e108af6eca9904d177d769281907d12ac6894dfc Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 16 Mar 2023 16:41:10 +0000 Subject: [PATCH 27/34] gh-102192: remove redundant exception fields from ssl module socket (#102466) --- Modules/_ssl.c | 28 +++++++++------------------- Modules/_ssl/debughelpers.c | 7 +++---- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 28112317bc289e..121d18884d0a9f 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -318,9 +318,7 @@ typedef struct { * store exception information on the socket. The handshake, read, write, * and shutdown methods check for chained exceptions. */ - PyObject *exc_type; - PyObject *exc_value; - PyObject *exc_tb; + PyObject *exc; } PySSLSocket; typedef struct { @@ -564,13 +562,11 @@ fill_and_set_sslerror(_sslmodulestate *state, static int PySSL_ChainExceptions(PySSLSocket *sslsock) { - if (sslsock->exc_type == NULL) + if (sslsock->exc == NULL) return 0; - _PyErr_ChainExceptions(sslsock->exc_type, sslsock->exc_value, sslsock->exc_tb); - sslsock->exc_type = NULL; - sslsock->exc_value = NULL; - sslsock->exc_tb = NULL; + _PyErr_ChainExceptions1(sslsock->exc); + sslsock->exc = NULL; return -1; } @@ -807,9 +803,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->owner = NULL; self->server_hostname = NULL; self->err = err; - self->exc_type = NULL; - self->exc_value = NULL; - self->exc_tb = NULL; + self->exc = NULL; /* Make sure the SSL error state is initialized */ ERR_clear_error(); @@ -2179,9 +2173,7 @@ Passed as \"self\" in servername callback."); static int PySSL_traverse(PySSLSocket *self, visitproc visit, void *arg) { - Py_VISIT(self->exc_type); - Py_VISIT(self->exc_value); - Py_VISIT(self->exc_tb); + Py_VISIT(self->exc); Py_VISIT(Py_TYPE(self)); return 0; } @@ -2189,9 +2181,7 @@ PySSL_traverse(PySSLSocket *self, visitproc visit, void *arg) static int PySSL_clear(PySSLSocket *self) { - Py_CLEAR(self->exc_type); - Py_CLEAR(self->exc_value); - Py_CLEAR(self->exc_tb); + Py_CLEAR(self->exc); return 0; } @@ -2536,7 +2526,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, PySSL_SetError(self, retval, __FILE__, __LINE__); goto error; } - if (self->exc_type != NULL) + if (self->exc != NULL) goto error; done: @@ -2662,7 +2652,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) PySSL_SetError(self, ret, __FILE__, __LINE__); return NULL; } - if (self->exc_type != NULL) + if (self->exc != NULL) goto error; if (sock) /* It's already INCREF'ed */ diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 08f3457035b90c..217f224942556e 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -74,7 +74,7 @@ _PySSL_msg_callback(int write_p, int version, int content_type, buf, len ); if (res == NULL) { - PyErr_Fetch(&ssl_obj->exc_type, &ssl_obj->exc_value, &ssl_obj->exc_tb); + ssl_obj->exc = PyErr_GetRaisedException(); } else { Py_DECREF(res); } @@ -138,8 +138,7 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) lock = PyThread_allocate_lock(); if (lock == NULL) { PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - PyErr_Fetch(&ssl_obj->exc_type, &ssl_obj->exc_value, - &ssl_obj->exc_tb); + ssl_obj->exc = PyErr_GetRaisedException(); return; } } @@ -156,7 +155,7 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) errno = e; PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, ssl_obj->ctx->keylog_filename); - PyErr_Fetch(&ssl_obj->exc_type, &ssl_obj->exc_value, &ssl_obj->exc_tb); + ssl_obj->exc = PyErr_GetRaisedException(); } PyGILState_Release(threadstate); } From 0f175766e27642108c65bba04bbd54dcf8799b0e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 16 Mar 2023 17:27:21 +0000 Subject: [PATCH 28/34] gh-99726: Improves correctness of stat results for Windows, and uses faster API when available (GH-102149) This deprecates `st_ctime` fields on Windows, with the intent to change them to contain the correct value in 3.14. For now, they should keep returning the creation time as they always have. --- Doc/library/os.rst | 88 +++++--- Doc/whatsnew/3.12.rst | 16 ++ Include/internal/pycore_fileutils.h | 5 +- Include/internal/pycore_fileutils_windows.h | 80 ++++++++ Lib/test/test_os.py | 12 +- ...3-02-22-17-26-10.gh-issue-99726.76t957.rst | 2 + Modules/posixmodule.c | 191 +++++++++++++----- PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/fileutils.c | 130 ++++++++++-- 10 files changed, 446 insertions(+), 82 deletions(-) create mode 100644 Include/internal/pycore_fileutils_windows.h create mode 100644 Misc/NEWS.d/next/Windows/2023-02-22-17-26-10.gh-issue-99726.76t957.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 5b9f49be1fad55..3153f79e10ce1f 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2858,6 +2858,12 @@ features: Added support for the :class:`~os.PathLike` interface. Added support for :class:`bytes` paths on Windows. + .. versionchanged:: 3.12 + The ``st_ctime`` attribute of a stat result is deprecated on Windows. + The file creation time is properly available as ``st_birthtime``, and + in the future ``st_ctime`` may be changed to return zero or the + metadata change time, if available. + .. function:: stat(path, *, dir_fd=None, follow_symlinks=True) @@ -2973,10 +2979,12 @@ features: .. attribute:: st_ctime - Platform dependent: + Time of most recent metadata change expressed in seconds. - * the time of most recent metadata change on Unix, - * the time of creation on Windows, expressed in seconds. + .. versionchanged:: 3.12 + ``st_ctime`` is deprecated on Windows. Use ``st_birthtime`` for + the file creation time. In the future, ``st_ctime`` will contain + the time of the most recent metadata change, as for other platforms. .. attribute:: st_atime_ns @@ -2989,29 +2997,48 @@ features: .. attribute:: st_ctime_ns - Platform dependent: + Time of most recent metadata change expressed in nanoseconds as an + integer. + + .. versionchanged:: 3.12 + ``st_ctime_ns`` is deprecated on Windows. Use ``st_birthtime_ns`` + for the file creation time. In the future, ``st_ctime`` will contain + the time of the most recent metadata change, as for other platforms. + + .. attribute:: st_birthtime + + Time of file creation expressed in seconds. This attribute is not + always available, and may raise :exc:`AttributeError`. + + .. versionchanged:: 3.12 + ``st_birthtime`` is now available on Windows. + + .. attribute:: st_birthtime_ns - * the time of most recent metadata change on Unix, - * the time of creation on Windows, expressed in nanoseconds as an - integer. + Time of file creation expressed in nanoseconds as an integer. + This attribute is not always available, and may raise + :exc:`AttributeError`. + + .. versionadded:: 3.12 .. note:: The exact meaning and resolution of the :attr:`st_atime`, - :attr:`st_mtime`, and :attr:`st_ctime` attributes depend on the operating - system and the file system. For example, on Windows systems using the FAT - or FAT32 file systems, :attr:`st_mtime` has 2-second resolution, and - :attr:`st_atime` has only 1-day resolution. See your operating system - documentation for details. + :attr:`st_mtime`, :attr:`st_ctime` and :attr:`st_birthtime` attributes + depend on the operating system and the file system. For example, on + Windows systems using the FAT32 file systems, :attr:`st_mtime` has + 2-second resolution, and :attr:`st_atime` has only 1-day resolution. + See your operating system documentation for details. Similarly, although :attr:`st_atime_ns`, :attr:`st_mtime_ns`, - and :attr:`st_ctime_ns` are always expressed in nanoseconds, many - systems do not provide nanosecond precision. On systems that do - provide nanosecond precision, the floating-point object used to - store :attr:`st_atime`, :attr:`st_mtime`, and :attr:`st_ctime` - cannot preserve all of it, and as such will be slightly inexact. - If you need the exact timestamps you should always use - :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and :attr:`st_ctime_ns`. + :attr:`st_ctime_ns` and :attr:`st_birthtime_ns` are always expressed in + nanoseconds, many systems do not provide nanosecond precision. On + systems that do provide nanosecond precision, the floating-point object + used to store :attr:`st_atime`, :attr:`st_mtime`, :attr:`st_ctime` and + :attr:`st_birthtime` cannot preserve all of it, and as such will be + slightly inexact. If you need the exact timestamps you should always use + :attr:`st_atime_ns`, :attr:`st_mtime_ns`, :attr:`st_ctime_ns` and + :attr:`st_birthtime_ns`. On some Unix systems (such as Linux), the following attributes may also be available: @@ -3041,10 +3068,6 @@ features: File generation number. - .. attribute:: st_birthtime - - Time of file creation. - On Solaris and derivatives, the following attributes may also be available: @@ -3117,6 +3140,25 @@ features: files as :const:`S_IFCHR`, :const:`S_IFIFO` or :const:`S_IFBLK` as appropriate. + .. versionchanged:: 3.12 + On Windows, :attr:`st_ctime` is deprecated. Eventually, it will + contain the last metadata change time, for consistency with other + platforms, but for now still contains creation time. + Use :attr:`st_birthtime` for the creation time. + + .. versionchanged:: 3.12 + On Windows, :attr:`st_ino` may now be up to 128 bits, depending + on the file system. Previously it would not be above 64 bits, and + larger file identifiers would be arbitrarily packed. + + .. versionchanged:: 3.12 + On Windows, :attr:`st_rdev` no longer returns a value. Previously + it would contain the same as :attr:`st_dev`, which was incorrect. + + .. versionadded:: 3.12 + Added the :attr:`st_birthtime` member on Windows. + + .. function:: statvfs(path) Perform a :c:func:`statvfs` system call on the given path. The return value is diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index b9c668543e1b73..03b1f975746787 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -306,6 +306,16 @@ os functions on Windows for enumerating drives, volumes and mount points. (Contributed by Steve Dower in :gh:`102519`.) +* :func:`os.stat` and :func:`os.lstat` are now more accurate on Windows. + The ``st_birthtime`` field will now be filled with the creation time + of the file, and ``st_ctime`` is deprecated but still contains the + creation time (but in the future will return the last metadata change, + for consistency with other platforms). ``st_dev`` may be up to 64 bits + and ``st_ino`` up to 128 bits depending on your file system, and + ``st_rdev`` is always set to zero rather than incorrect values. + Both functions may be significantly faster on newer releases of + Windows. (Contributed by Steve Dower in :gh:`99726`.) + os.path ------- @@ -469,6 +479,12 @@ Deprecated warning at compile time. This field will be removed in Python 3.14. (Contributed by Ramvikrams and Kumar Aditya in :gh:`101193`. PEP by Ken Jin.) +* The ``st_ctime`` fields return by :func:`os.stat` and :func:`os.lstat` on + Windows are deprecated. In a future release, they will contain the last + metadata change time, consistent with other platforms. For now, they still + contain the creation time, which is also available in the new ``st_birthtime`` + field. (Contributed by Steve Dower in :gh:`99726`.) + Pending Removal in Python 3.13 ------------------------------ diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 445ac0a3d955d9..ef6642d00f1b54 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -66,7 +66,7 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #ifdef MS_WINDOWS struct _Py_stat_struct { - unsigned long st_dev; + uint64_t st_dev; uint64_t st_ino; unsigned short st_mode; int st_nlink; @@ -80,8 +80,11 @@ struct _Py_stat_struct { int st_mtime_nsec; time_t st_ctime; int st_ctime_nsec; + time_t st_birthtime; + int st_birthtime_nsec; unsigned long st_file_attributes; unsigned long st_reparse_tag; + uint64_t st_ino_high; }; #else # define _Py_stat_struct stat diff --git a/Include/internal/pycore_fileutils_windows.h b/Include/internal/pycore_fileutils_windows.h new file mode 100644 index 00000000000000..44874903b092f3 --- /dev/null +++ b/Include/internal/pycore_fileutils_windows.h @@ -0,0 +1,80 @@ +#ifndef Py_INTERNAL_FILEUTILS_WINDOWS_H +#define Py_INTERNAL_FILEUTILS_WINDOWS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "Py_BUILD_CORE must be defined to include this header" +#endif + +#ifdef MS_WINDOWS + +#if !defined(NTDDI_WIN10_NI) || !(NTDDI_VERSION >= NTDDI_WIN10_NI) +typedef struct _FILE_STAT_BASIC_INFORMATION { + LARGE_INTEGER FileId; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER AllocationSize; + LARGE_INTEGER EndOfFile; + ULONG FileAttributes; + ULONG ReparseTag; + ULONG NumberOfLinks; + ULONG DeviceType; + ULONG DeviceCharacteristics; + ULONG Reserved; + FILE_ID_128 FileId128; + LARGE_INTEGER VolumeSerialNumber; +} FILE_STAT_BASIC_INFORMATION; + +typedef enum _FILE_INFO_BY_NAME_CLASS { + FileStatByNameInfo, + FileStatLxByNameInfo, + FileCaseSensitiveByNameInfo, + FileStatBasicByNameInfo, + MaximumFileInfoByNameClass +} FILE_INFO_BY_NAME_CLASS; +#endif + +typedef BOOL (WINAPI *PGetFileInformationByName)( + PCWSTR FileName, + FILE_INFO_BY_NAME_CLASS FileInformationClass, + PVOID FileInfoBuffer, + ULONG FileInfoBufferSize +); + +static inline BOOL _Py_GetFileInformationByName( + PCWSTR FileName, + FILE_INFO_BY_NAME_CLASS FileInformationClass, + PVOID FileInfoBuffer, + ULONG FileInfoBufferSize +) { + static PGetFileInformationByName GetFileInformationByName = NULL; + static int GetFileInformationByName_init = -1; + + if (GetFileInformationByName_init < 0) { + HMODULE hMod = LoadLibraryW(L"api-ms-win-core-file-l2-1-4"); + GetFileInformationByName_init = 0; + if (hMod) { + GetFileInformationByName = (PGetFileInformationByName)GetProcAddress( + hMod, "GetFileInformationByName"); + if (GetFileInformationByName) { + GetFileInformationByName_init = 1; + } else { + FreeLibrary(hMod); + } + } + } + + if (GetFileInformationByName_init <= 0) { + SetLastError(ERROR_NOT_SUPPORTED); + return FALSE; + } + return GetFileInformationByName(FileName, FileInformationClass, FileInfoBuffer, FileInfoBufferSize); +} + +#endif + +#endif diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 42357fef80ec89..74ece3ffb4ed17 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -556,6 +556,15 @@ def trunc(x): return x nanosecondy = getattr(result, name + "_ns") // 10000 self.assertAlmostEqual(floaty, nanosecondy, delta=2) + # Ensure both birthtime and birthtime_ns roughly agree, if present + try: + floaty = int(result.st_birthtime * 100000) + nanosecondy = result.st_birthtime_ns // 10000 + except AttributeError: + pass + else: + self.assertAlmostEqual(floaty, nanosecondy, delta=2) + try: result[200] self.fail("No exception raised") @@ -4234,7 +4243,8 @@ def assert_stat_equal(self, stat1, stat2, skip_fields): for attr in dir(stat1): if not attr.startswith("st_"): continue - if attr in ("st_dev", "st_ino", "st_nlink"): + if attr in ("st_dev", "st_ino", "st_nlink", "st_ctime", + "st_ctime_ns"): continue self.assertEqual(getattr(stat1, attr), getattr(stat2, attr), diff --git a/Misc/NEWS.d/next/Windows/2023-02-22-17-26-10.gh-issue-99726.76t957.rst b/Misc/NEWS.d/next/Windows/2023-02-22-17-26-10.gh-issue-99726.76t957.rst new file mode 100644 index 00000000000000..e2578620017894 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-22-17-26-10.gh-issue-99726.76t957.rst @@ -0,0 +1,2 @@ +Improves correctness of stat results for Windows, and uses faster API when +available diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7d91f7e4bac76b..e38caf7cc0abee 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -41,6 +41,7 @@ #ifndef MS_WINDOWS # include "posixmodule.h" #else +# include "pycore_fileutils_windows.h" # include "winreparse.h" #endif @@ -668,8 +669,11 @@ PyOS_AfterFork(void) #ifdef MS_WINDOWS /* defined in fileutils.c */ void _Py_time_t_to_FILE_TIME(time_t, int, FILETIME *); -void _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *, - ULONG, struct _Py_stat_struct *); +void _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *, ULONG, + FILE_BASIC_INFO *, FILE_ID_INFO *, + struct _Py_stat_struct *); +void _Py_stat_basic_info_to_stat(FILE_STAT_BASIC_INFORMATION *, + struct _Py_stat_struct *); #endif @@ -1819,12 +1823,39 @@ attributes_from_dir(LPCWSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *re return TRUE; } + +static void +update_st_mode_from_path(const wchar_t *path, DWORD attr, + struct _Py_stat_struct *result) +{ + if (!(attr & FILE_ATTRIBUTE_DIRECTORY)) { + /* Fix the file execute permissions. This hack sets S_IEXEC if + the filename has an extension that is commonly used by files + that CreateProcessW can execute. A real implementation calls + GetSecurityInfo, OpenThreadToken/OpenProcessToken, and + AccessCheck to check for generic read, write, and execute + access. */ + const wchar_t *fileExtension = wcsrchr(path, '.'); + if (fileExtension) { + if (_wcsicmp(fileExtension, L".exe") == 0 || + _wcsicmp(fileExtension, L".bat") == 0 || + _wcsicmp(fileExtension, L".cmd") == 0 || + _wcsicmp(fileExtension, L".com") == 0) { + result->st_mode |= 0111; + } + } + } +} + + static int -win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, - BOOL traverse) +win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, + BOOL traverse) { HANDLE hFile; BY_HANDLE_FILE_INFORMATION fileInfo; + FILE_BASIC_INFO basicInfo; + FILE_ID_INFO idInfo; FILE_ATTRIBUTE_TAG_INFO tagInfo = { 0 }; DWORD fileType, error; BOOL isUnhandledTag = FALSE; @@ -1954,12 +1985,16 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, for an unhandled tag. */ } else if (!isUnhandledTag) { CloseHandle(hFile); - return win32_xstat_impl(path, result, TRUE); + return win32_xstat_slow_impl(path, result, TRUE); } } } - if (!GetFileInformationByHandle(hFile, &fileInfo)) { + if (!GetFileInformationByHandle(hFile, &fileInfo) || + !GetFileInformationByHandleEx(hFile, FileBasicInfo, + &basicInfo, sizeof(basicInfo)) || + !GetFileInformationByHandleEx(hFile, FileIdInfo, + &idInfo, sizeof(idInfo))) { switch (GetLastError()) { case ERROR_INVALID_PARAMETER: case ERROR_INVALID_FUNCTION: @@ -1975,25 +2010,8 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, } } - _Py_attribute_data_to_stat(&fileInfo, tagInfo.ReparseTag, result); - - if (!(fileInfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) { - /* Fix the file execute permissions. This hack sets S_IEXEC if - the filename has an extension that is commonly used by files - that CreateProcessW can execute. A real implementation calls - GetSecurityInfo, OpenThreadToken/OpenProcessToken, and - AccessCheck to check for generic read, write, and execute - access. */ - const wchar_t *fileExtension = wcsrchr(path, '.'); - if (fileExtension) { - if (_wcsicmp(fileExtension, L".exe") == 0 || - _wcsicmp(fileExtension, L".bat") == 0 || - _wcsicmp(fileExtension, L".cmd") == 0 || - _wcsicmp(fileExtension, L".com") == 0) { - result->st_mode |= 0111; - } - } - } + _Py_attribute_data_to_stat(&fileInfo, tagInfo.ReparseTag, &basicInfo, &idInfo, result); + update_st_mode_from_path(path, fileInfo.dwFileAttributes, result); cleanup: if (hFile != INVALID_HANDLE_VALUE) { @@ -2010,6 +2028,39 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, return retval; } +static int +win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, + BOOL traverse) +{ + FILE_STAT_BASIC_INFORMATION statInfo; + if (_Py_GetFileInformationByName(path, FileStatBasicByNameInfo, + &statInfo, sizeof(statInfo))) { + if (// Cannot use fast path for reparse points ... + !(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + // ... unless it's a name surrogate (symlink) and we're not following + || (!traverse && IsReparseTagNameSurrogate(statInfo.ReparseTag)) + ) { + _Py_stat_basic_info_to_stat(&statInfo, result); + update_st_mode_from_path(path, statInfo.FileAttributes, result); + return 0; + } + } else { + switch(GetLastError()) { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_NOT_READY: + case ERROR_BAD_NET_NAME: + /* These errors aren't worth retrying with the slow path */ + return -1; + case ERROR_NOT_SUPPORTED: + /* indicates the API couldn't be loaded */ + break; + } + } + + return win32_xstat_slow_impl(path, result, traverse); +} + static int win32_xstat(const wchar_t *path, struct _Py_stat_struct *result, BOOL traverse) { @@ -2017,6 +2068,10 @@ win32_xstat(const wchar_t *path, struct _Py_stat_struct *result, BOOL traverse) setting it to a POSIX error. Callers should use GetLastError. */ int code = win32_xstat_impl(path, result, traverse); errno = 0; + + /* ctime is only deprecated from 3.12, so we copy birthtime across */ + result->st_ctime = result->st_birthtime; + result->st_ctime_nsec = result->st_birthtime_nsec; return code; } /* About the following functions: win32_lstat_w, win32_stat, win32_stat_w @@ -2087,9 +2142,12 @@ static PyStructSequence_Field stat_result_fields[] = { #ifdef HAVE_STRUCT_STAT_ST_GEN {"st_gen", "generation number"}, #endif -#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME +#if defined(HAVE_STRUCT_STAT_ST_BIRTHTIME) || defined(MS_WINDOWS) {"st_birthtime", "time of creation"}, #endif +#ifdef MS_WINDOWS + {"st_birthtime_ns", "time of creation in nanoseconds"}, +#endif #ifdef HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES {"st_file_attributes", "Windows file attribute bits"}, #endif @@ -2132,16 +2190,22 @@ static PyStructSequence_Field stat_result_fields[] = { #define ST_GEN_IDX ST_FLAGS_IDX #endif -#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME +#if defined(HAVE_STRUCT_STAT_ST_BIRTHTIME) || defined(MS_WINDOWS) #define ST_BIRTHTIME_IDX (ST_GEN_IDX+1) #else #define ST_BIRTHTIME_IDX ST_GEN_IDX #endif -#ifdef HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES -#define ST_FILE_ATTRIBUTES_IDX (ST_BIRTHTIME_IDX+1) +#ifdef MS_WINDOWS +#define ST_BIRTHTIME_NS_IDX (ST_BIRTHTIME_IDX+1) #else -#define ST_FILE_ATTRIBUTES_IDX ST_BIRTHTIME_IDX +#define ST_BIRTHTIME_NS_IDX ST_BIRTHTIME_IDX +#endif + +#if defined(HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES) || defined(MS_WINDOWS) +#define ST_FILE_ATTRIBUTES_IDX (ST_BIRTHTIME_NS_IDX+1) +#else +#define ST_FILE_ATTRIBUTES_IDX ST_BIRTHTIME_NS_IDX #endif #ifdef HAVE_STRUCT_STAT_ST_FSTYPE @@ -2310,7 +2374,7 @@ _posix_free(void *module) } static void -fill_time(PyObject *module, PyObject *v, int index, time_t sec, unsigned long nsec) +fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, time_t sec, unsigned long nsec) { PyObject *s = _PyLong_FromTime_t(sec); PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); @@ -2334,12 +2398,18 @@ fill_time(PyObject *module, PyObject *v, int index, time_t sec, unsigned long ns goto exit; } - PyStructSequence_SET_ITEM(v, index, s); - PyStructSequence_SET_ITEM(v, index+3, float_s); - PyStructSequence_SET_ITEM(v, index+6, ns_total); - s = NULL; - float_s = NULL; - ns_total = NULL; + if (s_index >= 0) { + PyStructSequence_SET_ITEM(v, s_index, s); + s = NULL; + } + if (f_index >= 0) { + PyStructSequence_SET_ITEM(v, f_index, float_s); + float_s = NULL; + } + if (ns_index >= 0) { + PyStructSequence_SET_ITEM(v, ns_index, ns_total); + ns_total = NULL; + } exit: Py_XDECREF(s); Py_XDECREF(ns_fractional); @@ -2348,6 +2418,33 @@ fill_time(PyObject *module, PyObject *v, int index, time_t sec, unsigned long ns Py_XDECREF(float_s); } +#ifdef MS_WINDOWS +static PyObject* +_pystat_l128_from_l64_l64(uint64_t low, uint64_t high) +{ + PyObject *o_low = PyLong_FromUnsignedLongLong(low); + if (!o_low || !high) { + return o_low; + } + PyObject *o_high = PyLong_FromUnsignedLongLong(high); + PyObject *l64 = o_high ? PyLong_FromLong(64) : NULL; + if (!l64) { + Py_XDECREF(o_high); + Py_DECREF(o_low); + return NULL; + } + Py_SETREF(o_high, PyNumber_Lshift(o_high, l64)); + Py_DECREF(l64); + if (!o_high) { + Py_DECREF(o_low); + return NULL; + } + Py_SETREF(o_low, PyNumber_Add(o_low, o_high)); + Py_DECREF(o_high); + return o_low; +} +#endif + /* pack a system stat C structure into the Python stat tuple (used by posix_stat() and posix_fstat()) */ static PyObject* @@ -2360,12 +2457,13 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) return NULL; PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long)st->st_mode)); +#ifdef MS_WINDOWS + PyStructSequence_SET_ITEM(v, 1, _pystat_l128_from_l64_l64(st->st_ino, st->st_ino_high)); + PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLongLong(st->st_dev)); +#else static_assert(sizeof(unsigned long long) >= sizeof(st->st_ino), "stat.st_ino is larger than unsigned long long"); PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLongLong(st->st_ino)); -#ifdef MS_WINDOWS - PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLong(st->st_dev)); -#else PyStructSequence_SET_ITEM(v, 2, _PyLong_FromDev(st->st_dev)); #endif PyStructSequence_SET_ITEM(v, 3, PyLong_FromLong((long)st->st_nlink)); @@ -2395,9 +2493,9 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) #else ansec = mnsec = cnsec = 0; #endif - fill_time(module, v, 7, st->st_atime, ansec); - fill_time(module, v, 8, st->st_mtime, mnsec); - fill_time(module, v, 9, st->st_ctime, cnsec); + fill_time(module, v, 7, 10, 13, st->st_atime, ansec); + fill_time(module, v, 8, 11, 14, st->st_mtime, mnsec); + fill_time(module, v, 9, 12, 15, st->st_ctime, cnsec); #ifdef HAVE_STRUCT_STAT_ST_BLKSIZE PyStructSequence_SET_ITEM(v, ST_BLKSIZE_IDX, @@ -2415,7 +2513,7 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) PyStructSequence_SET_ITEM(v, ST_GEN_IDX, PyLong_FromLong((long)st->st_gen)); #endif -#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME +#if defined(HAVE_STRUCT_STAT_ST_BIRTHTIME) { PyObject *val; unsigned long bsec,bnsec; @@ -2429,6 +2527,9 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) PyStructSequence_SET_ITEM(v, ST_BIRTHTIME_IDX, val); } +#elif defined(MS_WINDOWS) + fill_time(module, v, -1, ST_BIRTHTIME_IDX, ST_BIRTHTIME_NS_IDX, + st->st_birthtime, st->st_birthtime_nsec); #endif #ifdef HAVE_STRUCT_STAT_ST_FLAGS PyStructSequence_SET_ITEM(v, ST_FLAGS_IDX, @@ -14639,7 +14740,7 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) } find_data_to_file_info(dataW, &file_info, &reparse_tag); - _Py_attribute_data_to_stat(&file_info, reparse_tag, &entry->win32_lstat); + _Py_attribute_data_to_stat(&file_info, reparse_tag, NULL, NULL, &entry->win32_lstat); return (PyObject *)entry; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 85dc8caa458ed9..0343d30a42cd6a 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -216,6 +216,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 98e7d59ba1020c..359463e1d9af75 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -555,6 +555,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/fileutils.c b/Python/fileutils.c index f48b626b444016..969b7163b5ac18 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -8,6 +8,8 @@ #ifdef MS_WINDOWS # include # include +# include // FILE_DEVICE_* constants +# include "pycore_fileutils_windows.h" // FILE_STAT_BASIC_INFORMATION # if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) # define PATHCCH_ALLOW_LONG_PATHS 0x01 # else @@ -1056,6 +1058,13 @@ FILE_TIME_to_time_t_nsec(FILETIME *in_ptr, time_t *time_out, int* nsec_out) *time_out = Py_SAFE_DOWNCAST((in / 10000000) - secs_between_epochs, __int64, time_t); } +static void +LARGE_INTEGER_to_time_t_nsec(LARGE_INTEGER *in_ptr, time_t *time_out, int* nsec_out) +{ + *nsec_out = (int)(in_ptr->QuadPart % 10000000) * 100; /* FILETIME is in units of 100 nsec. */ + *time_out = Py_SAFE_DOWNCAST((in_ptr->QuadPart / 10000000) - secs_between_epochs, __int64, time_t); +} + void _Py_time_t_to_FILE_TIME(time_t time_in, int nsec_in, FILETIME *out_ptr) { @@ -1085,33 +1094,126 @@ attributes_to_mode(DWORD attr) return m; } + +typedef union { + FILE_ID_128 id; + struct { + uint64_t st_ino; + uint64_t st_ino_high; + }; +} id_128_to_ino; + + void _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, + FILE_BASIC_INFO *basic_info, FILE_ID_INFO *id_info, struct _Py_stat_struct *result) { memset(result, 0, sizeof(*result)); result->st_mode = attributes_to_mode(info->dwFileAttributes); result->st_size = (((__int64)info->nFileSizeHigh)<<32) + info->nFileSizeLow; - result->st_dev = info->dwVolumeSerialNumber; - result->st_rdev = result->st_dev; - FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_ctime, &result->st_ctime_nsec); - FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); - FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); + result->st_dev = id_info ? id_info->VolumeSerialNumber : info->dwVolumeSerialNumber; + result->st_rdev = 0; + /* st_ctime is deprecated, but we preserve the legacy value in our caller, not here */ + if (basic_info) { + LARGE_INTEGER_to_time_t_nsec(&basic_info->CreationTime, &result->st_birthtime, &result->st_birthtime_nsec); + LARGE_INTEGER_to_time_t_nsec(&basic_info->ChangeTime, &result->st_ctime, &result->st_ctime_nsec); + LARGE_INTEGER_to_time_t_nsec(&basic_info->LastWriteTime, &result->st_mtime, &result->st_mtime_nsec); + LARGE_INTEGER_to_time_t_nsec(&basic_info->LastAccessTime, &result->st_atime, &result->st_atime_nsec); + } else { + FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_birthtime, &result->st_birthtime_nsec); + FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); + FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); + } result->st_nlink = info->nNumberOfLinks; - result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow; + + if (id_info) { + id_128_to_ino file_id; + file_id.id = id_info->FileId; + result->st_ino = file_id.st_ino; + result->st_ino_high = file_id.st_ino_high; + } else { + /* should only occur for DirEntry_from_find_data, in which case the + index is likely to be zero anyway. */ + result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow; + } + /* bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will open other name surrogate reparse points without traversing them. To detect/handle these, check st_file_attributes and st_reparse_tag. */ result->st_reparse_tag = reparse_tag; if (info->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && reparse_tag == IO_REPARSE_TAG_SYMLINK) { - /* first clear the S_IFMT bits */ - result->st_mode ^= (result->st_mode & S_IFMT); - /* now set the bits that make this a symlink */ - result->st_mode |= S_IFLNK; + /* set the bits that make this a symlink */ + result->st_mode = (result->st_mode & ~S_IFMT) | S_IFLNK; } result->st_file_attributes = info->dwFileAttributes; } + +void +_Py_stat_basic_info_to_stat(FILE_STAT_BASIC_INFORMATION *info, + struct _Py_stat_struct *result) +{ + memset(result, 0, sizeof(*result)); + result->st_mode = attributes_to_mode(info->FileAttributes); + result->st_size = info->EndOfFile.QuadPart; + LARGE_INTEGER_to_time_t_nsec(&info->CreationTime, &result->st_birthtime, &result->st_birthtime_nsec); + LARGE_INTEGER_to_time_t_nsec(&info->ChangeTime, &result->st_ctime, &result->st_ctime_nsec); + LARGE_INTEGER_to_time_t_nsec(&info->LastWriteTime, &result->st_mtime, &result->st_mtime_nsec); + LARGE_INTEGER_to_time_t_nsec(&info->LastAccessTime, &result->st_atime, &result->st_atime_nsec); + result->st_nlink = info->NumberOfLinks; + result->st_dev = info->VolumeSerialNumber.QuadPart; + /* File systems with less than 128-bits zero pad into this field */ + id_128_to_ino file_id; + file_id.id = info->FileId128; + result->st_ino = file_id.st_ino; + result->st_ino_high = file_id.st_ino_high; + /* bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will + open other name surrogate reparse points without traversing them. To + detect/handle these, check st_file_attributes and st_reparse_tag. */ + result->st_reparse_tag = info->ReparseTag; + if (info->FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && + info->ReparseTag == IO_REPARSE_TAG_SYMLINK) { + /* set the bits that make this a symlink */ + result->st_mode = (result->st_mode & ~S_IFMT) | S_IFLNK; + } + result->st_file_attributes = info->FileAttributes; + switch (info->DeviceType) { + case FILE_DEVICE_DISK: + case FILE_DEVICE_VIRTUAL_DISK: + case FILE_DEVICE_DFS: + case FILE_DEVICE_CD_ROM: + case FILE_DEVICE_CONTROLLER: + case FILE_DEVICE_DATALINK: + break; + case FILE_DEVICE_DISK_FILE_SYSTEM: + case FILE_DEVICE_CD_ROM_FILE_SYSTEM: + case FILE_DEVICE_NETWORK_FILE_SYSTEM: + result->st_mode = (result->st_mode & ~S_IFMT) | 0x6000; /* _S_IFBLK */ + break; + case FILE_DEVICE_CONSOLE: + case FILE_DEVICE_NULL: + case FILE_DEVICE_KEYBOARD: + case FILE_DEVICE_MODEM: + case FILE_DEVICE_MOUSE: + case FILE_DEVICE_PARALLEL_PORT: + case FILE_DEVICE_PRINTER: + case FILE_DEVICE_SCREEN: + case FILE_DEVICE_SERIAL_PORT: + case FILE_DEVICE_SOUND: + result->st_mode = (result->st_mode & ~S_IFMT) | _S_IFCHR; + break; + case FILE_DEVICE_NAMED_PIPE: + result->st_mode = (result->st_mode & ~S_IFMT) | _S_IFIFO; + break; + default: + if (info->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + result->st_mode = (result->st_mode & ~S_IFMT) | _S_IFDIR; + } + break; + } +} + #endif /* Return information about a file. @@ -1131,6 +1233,8 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) { #ifdef MS_WINDOWS BY_HANDLE_FILE_INFORMATION info; + FILE_BASIC_INFO basicInfo; + FILE_ID_INFO idInfo; HANDLE h; int type; @@ -1162,14 +1266,16 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) return 0; } - if (!GetFileInformationByHandle(h, &info)) { + if (!GetFileInformationByHandle(h, &info) || + !GetFileInformationByHandleEx(h, FileBasicInfo, &basicInfo, sizeof(basicInfo)) || + !GetFileInformationByHandleEx(h, FileIdInfo, &idInfo, sizeof(idInfo))) { /* The Win32 error is already set, but we also set errno for callers who expect it */ errno = winerror_to_errno(GetLastError()); return -1; } - _Py_attribute_data_to_stat(&info, 0, status); + _Py_attribute_data_to_stat(&info, 0, &basicInfo, &idInfo, status); return 0; #else return fstat(fd, status); From 6372e290c0ac0993e4e2f66b12efdb076f620d09 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 16 Mar 2023 19:03:52 +0000 Subject: [PATCH 29/34] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives (#102760) --- Python/traceback.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Python/traceback.c b/Python/traceback.c index 31b85e77575efa..097f69c76abfe1 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -11,7 +11,7 @@ #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_parser.h" // _PyParser_ASTFromString #include "pycore_pyarena.h" // _PyArena_Free() -#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_traceback.h" // EXCEPTION_TB_HEADER @@ -242,17 +242,18 @@ _PyTraceBack_FromFrame(PyObject *tb_next, PyFrameObject *frame) int PyTraceBack_Here(PyFrameObject *frame) { - PyObject *exc, *val, *tb, *newtb; - PyErr_Fetch(&exc, &val, &tb); - newtb = _PyTraceBack_FromFrame(tb, frame); + PyObject *exc = PyErr_GetRaisedException(); + assert(PyExceptionInstance_Check(exc)); + PyObject *tb = PyException_GetTraceback(exc); + PyObject *newtb = _PyTraceBack_FromFrame(tb, frame); + Py_XDECREF(tb); if (newtb == NULL) { - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); return -1; } - assert(PyExceptionInstance_Check(val)); - PyException_SetTraceback(val, newtb); - PyErr_Restore(exc, val, newtb); - Py_XDECREF(tb); + PyException_SetTraceback(exc, newtb); + Py_XDECREF(newtb); + PyErr_SetRaisedException(exc); return 0; } From 405739f9166592104a5b0b945de92e28415ae972 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Thu, 16 Mar 2023 20:34:42 +0000 Subject: [PATCH 30/34] Fix outdated note about 'int' rounding or truncating (#102736) --- Doc/library/stdtypes.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 550f808a16dfaa..bcfc6e5cfce611 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -335,11 +335,10 @@ Notes: single: ceil() (in module math) single: trunc() (in module math) pair: numeric; conversions - pair: C; language - Conversion from floating point to integer may round or truncate - as in C; see functions :func:`math.floor` and :func:`math.ceil` for - well-defined conversions. + Conversion from :class:`float` to :class:`int` truncates, discarding the + fractional part. See functions :func:`math.floor` and :func:`math.ceil` for + alternative conversions. (4) float also accepts the strings "nan" and "inf" with an optional prefix "+" From 3f9285a8c52bf776c364f0cf4aecdd8f514ac4e1 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 16 Mar 2023 22:18:04 +0000 Subject: [PATCH 31/34] gh-102755: Add PyErr_DisplayException(exc) (#102756) --- Doc/c-api/exceptions.rst | 6 ++ Doc/data/stable_abi.dat | 1 + Doc/whatsnew/3.12.rst | 7 ++ Include/internal/pycore_pylifecycle.h | 1 + Include/pythonrun.h | 1 + Lib/test/test_stable_abi_ctypes.py | 1 + ...-03-16-14-44-29.gh-issue-102755.j1GxlV.rst | 3 + Misc/stable_abi.toml | 2 + Modules/_testcapi/exceptions.c | 13 +-- PC/python3dll.c | 1 + Python/pylifecycle.c | 29 ++----- Python/pythonrun.c | 87 +++++++++---------- Python/sysmodule.c | 2 +- 13 files changed, 76 insertions(+), 78 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-16-14-44-29.gh-issue-102755.j1GxlV.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8836c973f1f579..ddf7dc780b2745 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -86,6 +86,12 @@ Printing and clearing An exception must be set when calling this function. +.. c:function: void PyErr_DisplayException(PyObject *exc) + + Print the standard traceback display of ``exc`` to ``sys.stderr``, including + chained exceptions and notes. + + .. versionadded:: 3.12 Raising exceptions ================== diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 68ab0b5061f434..4cc06d22baaa93 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -133,6 +133,7 @@ function,PyErr_BadInternalCall,3.2,, function,PyErr_CheckSignals,3.2,, function,PyErr_Clear,3.2,, function,PyErr_Display,3.2,, +function,PyErr_DisplayException,3.12,, function,PyErr_ExceptionMatches,3.2,, function,PyErr_Fetch,3.2,, function,PyErr_Format,3.2,, diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 03b1f975746787..a398cd93f73120 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -944,6 +944,10 @@ New Features the :attr:`~BaseException.args` passed to the exception's constructor. (Contributed by Mark Shannon in :gh:`101578`.) +* Add :c:func:`PyErr_DisplayException`, which takes an exception instance, + to replace the legacy-api :c:func:`PyErr_Display`. (Contributed by + Irit Katriel in :gh:`102755`). + Porting to Python 3.12 ---------------------- @@ -1077,6 +1081,9 @@ Deprecated :c:func:`PyErr_SetRaisedException` instead. (Contributed by Mark Shannon in :gh:`101578`.) +* :c:func:`PyErr_Display` is deprecated. Use :c:func:`PyErr_DisplayException` + instead. (Contributed by Irit Katriel in :gh:`102755`). + Removed ------- diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index e7a31807205254..a899e848bb8b3c 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -87,6 +87,7 @@ PyAPI_FUNC(PyObject*) _PyErr_WriteUnraisableDefaultHook(PyObject *unraisable); PyAPI_FUNC(void) _PyErr_Print(PyThreadState *tstate); PyAPI_FUNC(void) _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *tb); +PyAPI_FUNC(void) _PyErr_DisplayException(PyObject *file, PyObject *exc); PyAPI_FUNC(void) _PyThreadState_DeleteCurrent(PyThreadState *tstate); diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 1b208b734ab1bf..41d82e89f84876 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -12,6 +12,7 @@ PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int); PyAPI_FUNC(void) PyErr_Print(void); PyAPI_FUNC(void) PyErr_PrintEx(int); PyAPI_FUNC(void) PyErr_Display(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(void) PyErr_DisplayException(PyObject *); /* Stuff with no proper home (yet) */ diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index e77c1c8409880d..2feaaf8603b831 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -166,6 +166,7 @@ def test_windows_feature_macros(self): "PyErr_CheckSignals", "PyErr_Clear", "PyErr_Display", + "PyErr_DisplayException", "PyErr_ExceptionMatches", "PyErr_Fetch", "PyErr_Format", diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-16-14-44-29.gh-issue-102755.j1GxlV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-16-14-44-29.gh-issue-102755.j1GxlV.rst new file mode 100644 index 00000000000000..d09af8d060d405 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-16-14-44-29.gh-issue-102755.j1GxlV.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyErr_DisplayException` which takes just an exception instance, +to replace the legacy :c:func:`PyErr_Display` which takes the ``(typ, exc, +tb)`` triplet. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 21ff9616133445..23baeeeae79193 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -609,6 +609,8 @@ added = '3.2' [function.PyErr_Display] added = '3.2' +[function.PyErr_DisplayException] + added = '3.12' [function.PyErr_ExceptionMatches] added = '3.2' [function.PyErr_Fetch] diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index c64b823663c32f..1922ca3beb7916 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -39,20 +39,13 @@ err_restore(PyObject *self, PyObject *args) { static PyObject * exception_print(PyObject *self, PyObject *args) { - PyObject *value; - PyObject *tb = NULL; + PyObject *exc; - if (!PyArg_ParseTuple(args, "O:exception_print", &value)) { + if (!PyArg_ParseTuple(args, "O:exception_print", &exc)) { return NULL; } - if (PyExceptionInstance_Check(value)) { - tb = PyException_GetTraceback(value); - } - - PyErr_Display((PyObject *) Py_TYPE(value), value, tb); - Py_XDECREF(tb); - + PyErr_DisplayException(exc); Py_RETURN_NONE; } diff --git a/PC/python3dll.c b/PC/python3dll.c index e300819365756e..706affa18351b3 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -192,6 +192,7 @@ EXPORT_FUNC(PyErr_BadInternalCall) EXPORT_FUNC(PyErr_CheckSignals) EXPORT_FUNC(PyErr_Clear) EXPORT_FUNC(PyErr_Display) +EXPORT_FUNC(PyErr_DisplayException) EXPORT_FUNC(PyErr_ExceptionMatches) EXPORT_FUNC(PyErr_Fetch) EXPORT_FUNC(PyErr_Format) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 82e94090a6027a..d0c65cc1f7fd44 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2537,41 +2537,28 @@ _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, static int _Py_FatalError_PrintExc(PyThreadState *tstate) { - PyObject *ferr, *res; - PyObject *exception, *v, *tb; - int has_tb; - - _PyErr_Fetch(tstate, &exception, &v, &tb); - if (exception == NULL) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (exc == NULL) { /* No current exception */ return 0; } - ferr = _PySys_GetAttr(tstate, &_Py_ID(stderr)); + PyObject *ferr = _PySys_GetAttr(tstate, &_Py_ID(stderr)); if (ferr == NULL || ferr == Py_None) { /* sys.stderr is not set yet or set to None, no need to try to display the exception */ return 0; } - _PyErr_NormalizeException(tstate, &exception, &v, &tb); - if (tb == NULL) { - tb = Py_NewRef(Py_None); - } - PyException_SetTraceback(v, tb); - if (exception == NULL) { - /* PyErr_NormalizeException() failed */ - return 0; - } + PyErr_DisplayException(exc); - has_tb = (tb != Py_None); - PyErr_Display(exception, v, tb); - Py_XDECREF(exception); - Py_XDECREF(v); + PyObject *tb = PyException_GetTraceback(exc); + int has_tb = (tb != NULL) && (tb != Py_None); Py_XDECREF(tb); + Py_XDECREF(exc); /* sys.stderr may be buffered: call sys.stderr.flush() */ - res = PyObject_CallMethodNoArgs(ferr, &_Py_ID(flush)); + PyObject *res = PyObject_CallMethodNoArgs(ferr, &_Py_ID(flush)); if (res == NULL) { _PyErr_Clear(tstate); } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 5381b105a39fed..07d119a67847c6 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -761,39 +761,34 @@ handle_system_exit(void) static void _PyErr_PrintEx(PyThreadState *tstate, int set_sys_last_vars) { - PyObject *exception, *v, *tb, *hook; - + PyObject *typ = NULL, *tb = NULL; handle_system_exit(); - _PyErr_Fetch(tstate, &exception, &v, &tb); - if (exception == NULL) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (exc == NULL) { goto done; } - - _PyErr_NormalizeException(tstate, &exception, &v, &tb); + assert(PyExceptionInstance_Check(exc)); + typ = Py_NewRef(Py_TYPE(exc)); + tb = PyException_GetTraceback(exc); if (tb == NULL) { tb = Py_NewRef(Py_None); } - PyException_SetTraceback(v, tb); - if (exception == NULL) { - goto done; - } - /* Now we know v != NULL too */ if (set_sys_last_vars) { - if (_PySys_SetAttr(&_Py_ID(last_type), exception) < 0) { + if (_PySys_SetAttr(&_Py_ID(last_type), typ) < 0) { _PyErr_Clear(tstate); } - if (_PySys_SetAttr(&_Py_ID(last_value), v) < 0) { + if (_PySys_SetAttr(&_Py_ID(last_value), exc) < 0) { _PyErr_Clear(tstate); } if (_PySys_SetAttr(&_Py_ID(last_traceback), tb) < 0) { _PyErr_Clear(tstate); } } - hook = _PySys_GetAttr(tstate, &_Py_ID(excepthook)); + PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(excepthook)); if (_PySys_Audit(tstate, "sys.excepthook", "OOOO", hook ? hook : Py_None, - exception, v, tb) < 0) { + typ, exc, tb) < 0) { if (PyErr_ExceptionMatches(PyExc_RuntimeError)) { PyErr_Clear(); goto done; @@ -802,46 +797,34 @@ _PyErr_PrintEx(PyThreadState *tstate, int set_sys_last_vars) } if (hook) { PyObject* stack[3]; - PyObject *result; - - stack[0] = exception; - stack[1] = v; + stack[0] = typ; + stack[1] = exc; stack[2] = tb; - result = _PyObject_FastCall(hook, stack, 3); + PyObject *result = _PyObject_FastCall(hook, stack, 3); if (result == NULL) { handle_system_exit(); - PyObject *exception2, *v2, *tb2; - _PyErr_Fetch(tstate, &exception2, &v2, &tb2); - _PyErr_NormalizeException(tstate, &exception2, &v2, &tb2); - /* It should not be possible for exception2 or v2 - to be NULL. However PyErr_Display() can't - tolerate NULLs, so just be safe. */ - if (exception2 == NULL) { - exception2 = Py_NewRef(Py_None); - } - if (v2 == NULL) { - v2 = Py_NewRef(Py_None); - } + PyObject *exc2 = _PyErr_GetRaisedException(tstate); + assert(exc2 && PyExceptionInstance_Check(exc2)); fflush(stdout); PySys_WriteStderr("Error in sys.excepthook:\n"); - PyErr_Display(exception2, v2, tb2); + PyErr_DisplayException(exc2); PySys_WriteStderr("\nOriginal exception was:\n"); - PyErr_Display(exception, v, tb); - Py_DECREF(exception2); - Py_DECREF(v2); - Py_XDECREF(tb2); + PyErr_DisplayException(exc); + Py_DECREF(exc2); + } + else { + Py_DECREF(result); } - Py_XDECREF(result); } else { PySys_WriteStderr("sys.excepthook is missing\n"); - PyErr_Display(exception, v, tb); + PyErr_DisplayException(exc); } done: - Py_XDECREF(exception); - Py_XDECREF(v); + Py_XDECREF(typ); + Py_XDECREF(exc); Py_XDECREF(tb); } @@ -1505,7 +1488,7 @@ print_exception_recursive(struct exception_print_context *ctx, PyObject *value) #define PyErr_MAX_GROUP_DEPTH 10 void -_PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *tb) +_PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) { assert(file != NULL && file != Py_None); if (PyExceptionInstance_Check(value) @@ -1513,10 +1496,12 @@ _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *t /* Put the traceback on the exception, otherwise it won't get displayed. See issue #18776. */ PyObject *cur_tb = PyException_GetTraceback(value); - if (cur_tb == NULL) + if (cur_tb == NULL) { PyException_SetTraceback(value, tb); - else + } + else { Py_DECREF(cur_tb); + } } struct exception_print_context ctx; @@ -1552,7 +1537,7 @@ _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *t } void -PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb) +PyErr_Display(PyObject *unused, PyObject *value, PyObject *tb) { PyThreadState *tstate = _PyThreadState_GET(); PyObject *file = _PySys_GetAttr(tstate, &_Py_ID(stderr)); @@ -1565,10 +1550,20 @@ PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb) return; } Py_INCREF(file); - _PyErr_Display(file, exception, value, tb); + _PyErr_Display(file, NULL, value, tb); Py_DECREF(file); } +void _PyErr_DisplayException(PyObject *file, PyObject *exc) +{ + _PyErr_Display(file, NULL, exc, NULL); +} + +void PyErr_DisplayException(PyObject *exc) +{ + PyErr_Display(NULL, exc, NULL); +} + PyObject * PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d282104dcad414..cc5b9a6d418bfa 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -742,7 +742,7 @@ sys_excepthook_impl(PyObject *module, PyObject *exctype, PyObject *value, PyObject *traceback) /*[clinic end generated code: output=18d99fdda21b6b5e input=ecf606fa826f19d9]*/ { - PyErr_Display(exctype, value, traceback); + PyErr_Display(NULL, value, traceback); Py_RETURN_NONE; } From f33b33eb31c11a32b2955eb1f002f02267bd7d61 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 17 Mar 2023 01:07:07 +0000 Subject: [PATCH 32/34] Increase stack reserve size for Windows debug builds to avoid test crashes (GH-102764) --- PCbuild/python.vcxproj | 2 +- PCbuild/pythonw.vcxproj | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index d07db3a6815058..f4640454a73070 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -95,7 +95,7 @@ Console 2000000 - 4000000 + 8000000 diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index e7216dec3a1af0..e23635e5ea9411 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -89,7 +89,8 @@ - 2000000 + 2000000 + 8000000 From 4f5774f648eafd1a7076ecf9af9629fb81baa363 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 17 Mar 2023 06:58:43 +0530 Subject: [PATCH 33/34] GH-78530: add support for generators in `asyncio.wait` (#102761) --- Doc/library/asyncio-task.rst | 4 ++++ Doc/whatsnew/3.12.rst | 3 +++ Lib/test/test_asyncio/test_tasks.py | 16 ++++++++++++++++ ...2023-03-16-16-43-04.gh-issue-78530.Lr8eq_.rst | 1 + 4 files changed, 24 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-03-16-16-43-04.gh-issue-78530.Lr8eq_.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 5f1449e1b599ef..a0900cd25a7731 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -804,6 +804,10 @@ Waiting Primitives .. versionchanged:: 3.11 Passing coroutine objects to ``wait()`` directly is forbidden. + .. versionchanged:: 3.12 + Added support for generators yielding tasks. + + .. function:: as_completed(aws, *, timeout=None) Run :ref:`awaitable objects ` in the *aws* diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a398cd93f73120..b55b9619fac226 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -241,6 +241,9 @@ asyncio :mod:`asyncio` does not support legacy generator-based coroutines. (Contributed by Kumar Aditya in :gh:`102748`.) +* :func:`asyncio.wait` now accepts generators yielding tasks. + (Contributed by Kumar Aditya in :gh:`78530`.) + inspect ------- diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 5b935b526541a1..731fa0c5a60b9b 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1373,6 +1373,22 @@ async def foo(): self.assertEqual(res, 42) self.assertAlmostEqual(0.15, loop.time()) + + def test_wait_generator(self): + async def func(a): + return a + + loop = self.new_test_loop() + + async def main(): + tasks = (self.new_task(loop, func(i)) for i in range(10)) + done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + self.assertEqual(len(done), 10) + self.assertEqual(len(pending), 0) + + loop.run_until_complete(main()) + + def test_as_completed(self): def gen(): diff --git a/Misc/NEWS.d/next/Library/2023-03-16-16-43-04.gh-issue-78530.Lr8eq_.rst b/Misc/NEWS.d/next/Library/2023-03-16-16-43-04.gh-issue-78530.Lr8eq_.rst new file mode 100644 index 00000000000000..bdb46d08c5c4af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-16-16-43-04.gh-issue-78530.Lr8eq_.rst @@ -0,0 +1 @@ +:func:`asyncio.wait` now accepts generators yielding tasks. Patch by Kumar Aditya. From 65fb7c4055f280caaa970939d16dd947e6df8a8d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 17 Mar 2023 22:39:09 +0900 Subject: [PATCH 34/34] gh-102701: Fix overflow in dictobject.c (GH-102750) --- Lib/test/test_bigmem.py | 9 +++++++++ .../2023-03-16-17-24-44.gh-issue-102701.iNGVaS.rst | 1 + Objects/dictobject.c | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-16-17-24-44.gh-issue-102701.iNGVaS.rst diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index 859f1539e20b80..c9ab1c1de9e186 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -1248,6 +1248,15 @@ def test_sort(self, size): self.assertEqual(l[-10:], [5] * 10) +class DictTest(unittest.TestCase): + + @bigmemtest(size=357913941, memuse=160) + def test_dict(self, size): + # https://github.com/python/cpython/issues/102701 + d = dict.fromkeys(range(size)) + d[size] = 1 + + if __name__ == '__main__': if len(sys.argv) > 1: support.set_memlimit(sys.argv[1]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-16-17-24-44.gh-issue-102701.iNGVaS.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-16-17-24-44.gh-issue-102701.iNGVaS.rst new file mode 100644 index 00000000000000..4e1f31893377ba --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-16-17-24-44.gh-issue-102701.iNGVaS.rst @@ -0,0 +1 @@ +Fix overflow when creating very large dict. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 227e438a8dfffc..53f9a380346a0d 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -596,7 +596,7 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) assert(log2_size >= PyDict_LOG_MINSIZE); - usable = USABLE_FRACTION(1<