From 2a03ed034edc6bb2db6fb972b2efe08db5145a28 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Mar 2023 18:43:54 -0600 Subject: [PATCH 1/5] gh-102660: Fix Refleaks in import.c (#102744) gh-102661 introduced some leaks. This fixes them. https://github.com/python/cpython/issues/102660 --- Python/bltinmodule.c | 3 --- Python/import.c | 50 +++++++++++++++++++++++++------------------- Python/sysmodule.c | 3 --- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index a8a34620b9bcdf..12ca0ba6c4873c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3098,9 +3098,6 @@ _PyBuiltin_Init(PyInterpreterState *interp) } Py_DECREF(debug); - /* m_copy of Py_None means it is copied some other way. */ - builtinsmodule.m_base.m_copy = Py_NewRef(Py_None); - return mod; #undef ADD_TO_ALL #undef SETBUILTIN diff --git a/Python/import.c b/Python/import.c index 612fee243bd9e7..9f80c6d8dd49a8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -978,14 +978,29 @@ _PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) return 0; } -static inline int -match_mod_name(PyObject *actual, const char *expected) +static PyObject * +get_core_module_dict(PyInterpreterState *interp, + PyObject *name, PyObject *filename) { - if (PyUnicode_CompareWithASCIIString(actual, expected) == 0) { - return 1; + /* Only builtin modules are core. */ + if (filename == name) { + assert(!PyErr_Occurred()); + if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { + return interp->sysdict_copy; + } + assert(!PyErr_Occurred()); + if (PyUnicode_CompareWithASCIIString(name, "builtins") == 0) { + return interp->builtins_copy; + } + assert(!PyErr_Occurred()); } - assert(!PyErr_Occurred()); - return 0; + return NULL; +} + +static inline int +is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *filename) +{ + return get_core_module_dict(interp, name, filename) != NULL; } static int @@ -1009,10 +1024,8 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) // bpo-44050: Extensions and def->m_base.m_copy can be updated // when the extension module doesn't support sub-interpreters. - // XXX Why special-case the main interpreter? - if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { - /* m_copy of Py_None means it is copied some other way. */ - if (def->m_size == -1 && def->m_base.m_copy != Py_None) { + if (def->m_size == -1) { + if (!is_core_module(tstate->interp, name, filename)) { if (def->m_base.m_copy) { /* Somebody already imported the module, likely under a different name. @@ -1028,7 +1041,10 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) return -1; } } + } + // XXX Why special-case the main interpreter? + if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { if (_extensions_cache_set(filename, name, def) < 0) { return -1; } @@ -1069,21 +1085,11 @@ import_find_extension(PyThreadState *tstate, PyObject *name, PyObject *m_copy = def->m_base.m_copy; /* Module does not support repeated initialization */ if (m_copy == NULL) { - return NULL; - } - else if (m_copy == Py_None) { - if (match_mod_name(name, "sys")) { - m_copy = tstate->interp->sysdict_copy; - } - else if (match_mod_name(name, "builtins")) { - m_copy = tstate->interp->builtins_copy; - } - else { - _PyErr_SetString(tstate, PyExc_ImportError, "missing m_copy"); + m_copy = get_core_module_dict(tstate->interp, name, filename); + if (m_copy == NULL) { return NULL; } } - /* m_copy of Py_None means it is copied some other way. */ mod = import_add_module(tstate, name); if (mod == NULL) { return NULL; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index fc0550266bf1af..d282104dcad414 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3425,9 +3425,6 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) return _PyStatus_ERR("failed to create a module object"); } - /* m_copy of Py_None means it is copied some other way. */ - sysmodule.m_base.m_copy = Py_NewRef(Py_None); - PyObject *sysdict = PyModule_GetDict(sysmod); if (sysdict == NULL) { goto error; From 1c9f3391b916939e5ad18213e553f8d6bfbec25e Mon Sep 17 00:00:00 2001 From: Jamoo721 <81095953+Jamoo721@users.noreply.github.com> Date: Thu, 16 Mar 2023 13:52:11 +1100 Subject: [PATCH 2/5] gh-102690: Use Edge as fallback in webbrowser instead of IE (#102691) --- Lib/webbrowser.py | 12 ++++++++---- .../2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 44974d433b4696..a56ff33dbbdc69 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -542,11 +542,15 @@ def register_standard_browsers(): # First try to use the default Windows browser register("windows-default", WindowsDefault) - # Detect some common Windows browsers, fallback to IE - iexplore = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"), - "Internet Explorer\\IEXPLORE.EXE") + # Detect some common Windows browsers, fallback to Microsoft Edge + # location in 64-bit Windows + edge64 = os.path.join(os.environ.get("PROGRAMFILES(x86)", "C:\\Program Files (x86)"), + "Microsoft\\Edge\\Application\\msedge.exe") + # location in 32-bit Windows + edge32 = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"), + "Microsoft\\Edge\\Application\\msedge.exe") for browser in ("firefox", "firebird", "seamonkey", "mozilla", - "netscape", "opera", iexplore): + "opera", edge64, edge32): if shutil.which(browser): register(browser, None, BackgroundBrowser(browser)) else: diff --git a/Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst b/Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst new file mode 100644 index 00000000000000..5669ebbb442c24 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst @@ -0,0 +1 @@ +Update :mod:`webbrowser` to fall back to Microsoft Edge instead of Internet Explorer. From a44553ea9f7745a1119148082edb1fb0372ac0e2 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 16 Mar 2023 09:20:43 +0530 Subject: [PATCH 3/5] GH-100112: avoid using iterable coroutines in asyncio internally (#100128) --- Lib/asyncio/tasks.py | 22 +++++-------------- Lib/test/test_asyncio/test_tasks.py | 15 +++++++++++++ ...-03-10-13-51-21.gh-issue-100112.VHh4mw.rst | 1 + 3 files changed, 22 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 1c20754b839b69..c90d32c97add78 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -25,7 +25,6 @@ from . import exceptions from . import futures from . import timeouts -from .coroutines import _is_coroutine # Helper to generate new task names # This uses itertools.count() instead of a "+= 1" operation because the latter @@ -635,11 +634,14 @@ def ensure_future(coro_or_future, *, loop=None): raise ValueError('The future belongs to a different loop than ' 'the one specified as the loop argument') return coro_or_future - called_wrap_awaitable = False + should_close = True if not coroutines.iscoroutine(coro_or_future): if inspect.isawaitable(coro_or_future): + async def _wrap_awaitable(awaitable): + return await awaitable + coro_or_future = _wrap_awaitable(coro_or_future) - called_wrap_awaitable = True + should_close = False else: raise TypeError('An asyncio.Future, a coroutine or an awaitable ' 'is required') @@ -649,23 +651,11 @@ def ensure_future(coro_or_future, *, loop=None): try: return loop.create_task(coro_or_future) except RuntimeError: - if not called_wrap_awaitable: + if should_close: coro_or_future.close() raise -@types.coroutine -def _wrap_awaitable(awaitable): - """Helper for asyncio.ensure_future(). - - Wraps awaitable (an object with __await__) into a coroutine - that will later be wrapped in a Task by ensure_future(). - """ - return (yield from awaitable.__await__()) - -_wrap_awaitable._is_coroutine = _is_coroutine - - class _GatheringFuture(futures.Future): """Helper for gather(). diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index e533d5273e9f38..5b935b526541a1 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -8,6 +8,7 @@ import re import sys import traceback +import types import unittest from unittest import mock from types import GenericAlias @@ -274,6 +275,20 @@ async def coro(): loop.run_until_complete(fut) self.assertEqual(fut.result(), 'ok') + def test_ensure_future_task_awaitable(self): + class Aw: + def __await__(self): + return asyncio.sleep(0, result='ok').__await__() + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + task = asyncio.ensure_future(Aw(), loop=loop) + loop.run_until_complete(task) + self.assertTrue(task.done()) + self.assertEqual(task.result(), 'ok') + self.assertIsInstance(task.get_coro(), types.CoroutineType) + loop.close() + def test_ensure_future_neither(self): with self.assertRaises(TypeError): asyncio.ensure_future('ok') diff --git a/Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst b/Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst new file mode 100644 index 00000000000000..eff77c40e30c48 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst @@ -0,0 +1 @@ +:meth:`asyncio.Task.get_coro` now always returns a coroutine when wrapping an awaitable object. Patch by Kumar Aditya. From 2dc94634b50f0e5e207787e5ac1d56c68b22c3ae Mon Sep 17 00:00:00 2001 From: yonatanp Date: Thu, 16 Mar 2023 00:44:52 -0400 Subject: [PATCH 4/5] gh-94440: Fix issue of ProcessPoolExecutor shutdown hanging (#94468) Fix an issue of concurrent.futures ProcessPoolExecutor shutdown hanging. Co-authored-by: Alex Waygood --- Lib/concurrent/futures/process.py | 5 ++++ Lib/test/test_concurrent_futures.py | 28 +++++++++++++++++++ Misc/ACKS | 1 + ...2-06-30-21-28-41.gh-issue-94440.LtgX0d.rst | 2 ++ 4 files changed, 36 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 3a8637b6fa1b47..816edab99f63e3 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -366,6 +366,11 @@ def run(self): if self.is_shutting_down(): self.flag_executor_shutting_down() + # When only canceled futures remain in pending_work_items, our + # next call to wait_result_broken_or_wakeup would hang forever. + # This makes sure we have some running futures or none at all. + self.add_call_item_to_queue() + # Since no new work items can be added, it is safe to shutdown # this thread if there are no pending work items. if not self.pending_work_items: diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index b3520ae3994e03..a20cb844a293c9 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -14,6 +14,7 @@ from logging.handlers import QueueHandler import os import queue +import signal import sys import threading import time @@ -397,6 +398,33 @@ def test_hang_gh83386(self): self.assertFalse(err) self.assertEqual(out.strip(), b"apple") + def test_hang_gh94440(self): + """shutdown(wait=True) doesn't hang when a future was submitted and + quickly canceled right before shutdown. + + See https://github.com/python/cpython/issues/94440. + """ + if not hasattr(signal, 'alarm'): + raise unittest.SkipTest( + "Tested platform does not support the alarm signal") + + def timeout(_signum, _frame): + raise RuntimeError("timed out waiting for shutdown") + + kwargs = {} + if getattr(self, 'ctx', None): + kwargs['mp_context'] = self.get_context() + executor = self.executor_type(max_workers=1, **kwargs) + executor.submit(int).result() + old_handler = signal.signal(signal.SIGALRM, timeout) + try: + signal.alarm(5) + executor.submit(int).cancel() + executor.shutdown(wait=True) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, BaseTestCase): def test_threads_terminate(self): diff --git a/Misc/ACKS b/Misc/ACKS index 7bbde3af99782b..8cf5166a2bb1f4 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1385,6 +1385,7 @@ Thomas Perl Mathieu Perreault Mark Perrego Trevor Perrin +Yonatan Perry Gabriel de Perthuis Tim Peters Benjamin Peterson diff --git a/Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst b/Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst new file mode 100644 index 00000000000000..3eee82e59dfafb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst @@ -0,0 +1,2 @@ +Fix a :mod:`concurrent.futures.process` bug where ``ProcessPoolExecutor`` shutdown +could hang after a future has been quickly submitted and canceled. From 51d693c58454a2c525094a7c74ebac86859353fd Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 16 Mar 2023 10:16:01 +0000 Subject: [PATCH 5/5] gh-102594: PyErr_SetObject adds note to exception raised on normalization error (#102675) --- Include/cpython/pyerrors.h | 4 ++ Lib/test/test_capi/test_exceptions.py | 20 ++++++++++ ...-03-14-00-11-46.gh-issue-102594.BjU-m2.rst | 1 + Modules/_testcapi/exceptions.c | 21 ++++++++++ Objects/exceptions.c | 15 +++++++ Python/errors.c | 40 ++++++++++++++++--- 6 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 0d9cc9922f7368..d0300f6ee56a25 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -112,6 +112,10 @@ PyAPI_FUNC(PyObject *) _PyErr_FormatFromCause( /* In exceptions.c */ +PyAPI_FUNC(int) _PyException_AddNote( + PyObject *exc, + PyObject *note); + /* Helper that attempts to replace the current exception with one of the * same type but with a prefix added to the exception text. The resulting * exception description looks like: diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index 55f131699a2567..b1c1a61e20685e 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -169,5 +169,25 @@ class Broken(Exception, metaclass=Meta): with self.assertRaises(ZeroDivisionError) as e: _testcapi.exc_set_object(Broken, Broken()) + def test_set_object_and_fetch(self): + class Broken(Exception): + def __init__(self, *arg): + raise ValueError("Broken __init__") + + exc = _testcapi.exc_set_object_fetch(Broken, 'abcd') + self.assertIsInstance(exc, ValueError) + self.assertEqual(exc.__notes__[0], + "Normalization failed: type=Broken args='abcd'") + + class BadArg: + def __repr__(self): + raise TypeError('Broken arg type') + + exc = _testcapi.exc_set_object_fetch(Broken, BadArg()) + self.assertIsInstance(exc, ValueError) + self.assertEqual(exc.__notes__[0], + 'Normalization failed: type=Broken args=') + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst new file mode 100644 index 00000000000000..0b95b5ec98e811 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst @@ -0,0 +1 @@ +Add note to exception raised in ``PyErr_SetObject`` when normalization fails. diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index a0575213987ffc..c64b823663c32f 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -92,6 +92,26 @@ exc_set_object(PyObject *self, PyObject *args) return NULL; } +static PyObject * +exc_set_object_fetch(PyObject *self, PyObject *args) +{ + PyObject *exc; + PyObject *obj; + PyObject *type; + PyObject *value; + PyObject *tb; + + if (!PyArg_ParseTuple(args, "OO:exc_set_object", &exc, &obj)) { + return NULL; + } + + PyErr_SetObject(exc, obj); + PyErr_Fetch(&type, &value, &tb); + Py_XDECREF(type); + Py_XDECREF(tb); + return value; +} + static PyObject * raise_exception(PyObject *self, PyObject *args) { @@ -262,6 +282,7 @@ static PyMethodDef test_methods[] = { {"make_exception_with_doc", _PyCFunction_CAST(make_exception_with_doc), METH_VARARGS | METH_KEYWORDS}, {"exc_set_object", exc_set_object, METH_VARARGS}, + {"exc_set_object_fetch", exc_set_object_fetch, METH_VARARGS}, {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", raise_memoryerror, METH_NOARGS}, {"set_exc_info", test_set_exc_info, METH_VARARGS}, diff --git a/Objects/exceptions.c b/Objects/exceptions.c index c6fb6a3f19b2d0..d69f7400ca6042 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3749,6 +3749,21 @@ _PyExc_Fini(PyInterpreterState *interp) _PyExc_FiniTypes(interp); } +int +_PyException_AddNote(PyObject *exc, PyObject *note) +{ + if (!PyExceptionInstance_Check(exc)) { + PyErr_Format(PyExc_TypeError, + "exc must be an exception, not '%s'", + Py_TYPE(exc)->tp_name); + return -1; + } + PyObject *r = BaseException_add_note(exc, note); + int res = r == NULL ? -1 : 0; + Py_XDECREF(r); + return res; +} + /* Helper to do the equivalent of "raise X from Y" in C, but always using * the current exception rather than passing one in. * diff --git a/Python/errors.c b/Python/errors.c index bbf6d397ce8097..bdcbac317eb9ee 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -135,6 +135,28 @@ _PyErr_GetTopmostException(PyThreadState *tstate) return exc_info; } +static PyObject * +get_normalization_failure_note(PyThreadState *tstate, PyObject *exception, PyObject *value) +{ + PyObject *args = PyObject_Repr(value); + if (args == NULL) { + _PyErr_Clear(tstate); + args = PyUnicode_FromFormat(""); + } + PyObject *note; + const char *tpname = ((PyTypeObject*)exception)->tp_name; + if (args == NULL) { + _PyErr_Clear(tstate); + note = PyUnicode_FromFormat("Normalization failed: type=%s", tpname); + } + else { + note = PyUnicode_FromFormat("Normalization failed: type=%s args=%S", + tpname, args); + Py_DECREF(args); + } + return note; +} + void _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) { @@ -160,19 +182,27 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) Py_XINCREF(value); if (!is_subclass) { /* We must normalize the value right now */ - PyObject *fixed_value; /* Issue #23571: functions must not be called with an exception set */ _PyErr_Clear(tstate); - fixed_value = _PyErr_CreateException(exception, value); - Py_XDECREF(value); + PyObject *fixed_value = _PyErr_CreateException(exception, value); if (fixed_value == NULL) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + assert(PyExceptionInstance_Check(exc)); + + PyObject *note = get_normalization_failure_note(tstate, exception, value); + Py_XDECREF(value); + if (note != NULL) { + /* ignore errors in _PyException_AddNote - they will be overwritten below */ + _PyException_AddNote(exc, note); + Py_DECREF(note); + } + _PyErr_SetRaisedException(tstate, exc); return; } - - value = fixed_value; + Py_XSETREF(value, fixed_value); } exc_value = _PyErr_GetTopmostException(tstate)->exc_value;