Merge remote-tracking branch 'upstream/main' into pythonrun

Include/cpython/pyerrors.h

@@ -112,6 +112,10 @@ PyAPI_FUNC(PyObject *) _PyErr_FormatFromCause(
 
 /* In exceptions.c */
 
+PyAPI_FUNC(int) _PyException_AddNote(
+ PyObject *exc,
+ PyObject *note);
+
 /* Helper that attempts to replace the current exception with one of the
  * same type but with a prefix added to the exception text. The resulting
  * exception description looks like:

Lib/asyncio/tasks.py

@@ -25,7 +25,6 @@
 from . import exceptions
 from . import futures
 from . import timeouts
-from .coroutines import _is_coroutine
 
 # Helper to generate new task names
 # This uses itertools.count() instead of a "+= 1" operation because the latter
@@ -635,11 +634,14 @@ def ensure_future(coro_or_future, *, loop=None):
  raise ValueError('The future belongs to a different loop than '
  'the one specified as the loop argument')
  return coro_or_future
- called_wrap_awaitable = False
+ should_close = True
  if not coroutines.iscoroutine(coro_or_future):
  if inspect.isawaitable(coro_or_future):
+ async def _wrap_awaitable(awaitable):
+ return await awaitable
+
  coro_or_future = _wrap_awaitable(coro_or_future)
- called_wrap_awaitable = True
+ should_close = False
  else:
  raise TypeError('An asyncio.Future, a coroutine or an awaitable '
  'is required')
@@ -649,23 +651,11 @@ def ensure_future(coro_or_future, *, loop=None):
  try:
  return loop.create_task(coro_or_future)
  except RuntimeError:
- if not called_wrap_awaitable:
+ if should_close:
  coro_or_future.close()
  raise
 
 
-@types.coroutine
-def _wrap_awaitable(awaitable):
- """Helper for asyncio.ensure_future().
-
- Wraps awaitable (an object with __await__) into a coroutine
- that will later be wrapped in a Task by ensure_future().
- """
- return (yield from awaitable.__await__())
-
-_wrap_awaitable._is_coroutine = _is_coroutine
-
-
 class _GatheringFuture(futures.Future):
  """Helper for gather().
 

Lib/concurrent/futures/process.py

@@ -366,6 +366,11 @@ def run(self):
  if self.is_shutting_down():
  self.flag_executor_shutting_down()
 
+ # When only canceled futures remain in pending_work_items, our
+ # next call to wait_result_broken_or_wakeup would hang forever.
+ # This makes sure we have some running futures or none at all.
+ self.add_call_item_to_queue()
+
  # Since no new work items can be added, it is safe to shutdown
  # this thread if there are no pending work items.
  if not self.pending_work_items:

Lib/test/test_asyncio/test_tasks.py

@@ -8,6 +8,7 @@
 import re
 import sys
 import traceback
+import types
 import unittest
 from unittest import mock
 from types import GenericAlias
@@ -274,6 +275,20 @@ async def coro():
  loop.run_until_complete(fut)
  self.assertEqual(fut.result(), 'ok')
 
+ def test_ensure_future_task_awaitable(self):
+ class Aw:
+ def __await__(self):
+ return asyncio.sleep(0, result='ok').__await__()
+
+ loop = asyncio.new_event_loop()
+ self.set_event_loop(loop)
+ task = asyncio.ensure_future(Aw(), loop=loop)
+ loop.run_until_complete(task)
+ self.assertTrue(task.done())
+ self.assertEqual(task.result(), 'ok')
+ self.assertIsInstance(task.get_coro(), types.CoroutineType)
+ loop.close()
+
  def test_ensure_future_neither(self):
  with self.assertRaises(TypeError):
  asyncio.ensure_future('ok')

Lib/test/test_capi/test_exceptions.py

@@ -169,5 +169,25 @@ class Broken(Exception, metaclass=Meta):
  with self.assertRaises(ZeroDivisionError) as e:
  _testcapi.exc_set_object(Broken, Broken())
 
+ def test_set_object_and_fetch(self):
+ class Broken(Exception):
+ def __init__(self, *arg):
+ raise ValueError("Broken __init__")
+
+ exc = _testcapi.exc_set_object_fetch(Broken, 'abcd')
+ self.assertIsInstance(exc, ValueError)
+ self.assertEqual(exc.__notes__[0],
+ "Normalization failed: type=Broken args='abcd'")
+
+ class BadArg:
+ def __repr__(self):
+ raise TypeError('Broken arg type')
+
+ exc = _testcapi.exc_set_object_fetch(Broken, BadArg())
+ self.assertIsInstance(exc, ValueError)
+ self.assertEqual(exc.__notes__[0],
+ 'Normalization failed: type=Broken args=<unknown>')
+
+
 if __name__ == "__main__":
  unittest.main()

Lib/test/test_concurrent_futures.py

@@ -14,6 +14,7 @@
 from logging.handlers import QueueHandler
 import os
 import queue
+import signal
 import sys
 import threading
 import time
@@ -397,6 +398,33 @@ def test_hang_gh83386(self):
  self.assertFalse(err)
  self.assertEqual(out.strip(), b"apple")
 
+ def test_hang_gh94440(self):
+ """shutdown(wait=True) doesn't hang when a future was submitted and
+ quickly canceled right before shutdown.
+
+ See https://github.com/python/cpython/issues/94440.
+ """
+ if not hasattr(signal, 'alarm'):
+ raise unittest.SkipTest(
+ "Tested platform does not support the alarm signal")
+
+ def timeout(_signum, _frame):
+ raise RuntimeError("timed out waiting for shutdown")
+
+ kwargs = {}
+ if getattr(self, 'ctx', None):
+ kwargs['mp_context'] = self.get_context()
+ executor = self.executor_type(max_workers=1, **kwargs)
+ executor.submit(int).result()
+ old_handler = signal.signal(signal.SIGALRM, timeout)
+ try:
+ signal.alarm(5)
+ executor.submit(int).cancel()
+ executor.shutdown(wait=True)
+ finally:
+ signal.alarm(0)
+ signal.signal(signal.SIGALRM, old_handler)
+
 
 class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, BaseTestCase):
  def test_threads_terminate(self):

Lib/webbrowser.py

@@ -542,11 +542,15 @@ def register_standard_browsers():
  # First try to use the default Windows browser
  register("windows-default", WindowsDefault)
 
- # Detect some common Windows browsers, fallback to IE
- iexplore = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"),
- "Internet Explorer\\IEXPLORE.EXE")
+ # Detect some common Windows browsers, fallback to Microsoft Edge
+ # location in 64-bit Windows
+ edge64 = os.path.join(os.environ.get("PROGRAMFILES(x86)", "C:\\Program Files (x86)"),
+ "Microsoft\\Edge\\Application\\msedge.exe")
+ # location in 32-bit Windows
+ edge32 = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"),
+ "Microsoft\\Edge\\Application\\msedge.exe")
  for browser in ("firefox", "firebird", "seamonkey", "mozilla",
- "netscape", "opera", iexplore):
+ "opera", edge64, edge32):
  if shutil.which(browser):
  register(browser, None, BackgroundBrowser(browser))
  else:

Misc/ACKS

@@ -1385,6 +1385,7 @@ Thomas Perl
 Mathieu Perreault
 Mark Perrego
 Trevor Perrin
+Yonatan Perry
 Gabriel de Perthuis
 Tim Peters
 Benjamin Peterson

Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst

@@ -0,0 +1 @@
+Add note to exception raised in ``PyErr_SetObject`` when normalization fails.

Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst

@@ -0,0 +1,2 @@
+Fix a :mod:`concurrent.futures.process` bug where ``ProcessPoolExecutor`` shutdown
+could hang after a future has been quickly submitted and canceled.

Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst

@@ -0,0 +1 @@
+:meth:`asyncio.Task.get_coro` now always returns a coroutine when wrapping an awaitable object. Patch by Kumar Aditya.

Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst

@@ -0,0 +1 @@
+Update :mod:`webbrowser` to fall back to Microsoft Edge instead of Internet Explorer.

Modules/_testcapi/exceptions.c

@@ -92,6 +92,26 @@ exc_set_object(PyObject *self, PyObject *args)
  return NULL;
 }
 
+static PyObject *
+exc_set_object_fetch(PyObject *self, PyObject *args)
+{
+ PyObject *exc;
+ PyObject *obj;
+ PyObject *type;
+ PyObject *value;
+ PyObject *tb;
+
+ if (!PyArg_ParseTuple(args, "OO:exc_set_object", &exc, &obj)) {
+ return NULL;
+ }
+
+ PyErr_SetObject(exc, obj);
+ PyErr_Fetch(&type, &value, &tb);
+ Py_XDECREF(type);
+ Py_XDECREF(tb);
+ return value;
+}
+
 static PyObject *
 raise_exception(PyObject *self, PyObject *args)
 {
@@ -262,6 +282,7 @@ static PyMethodDef test_methods[] = {
  {"make_exception_with_doc", _PyCFunction_CAST(make_exception_with_doc),
  METH_VARARGS | METH_KEYWORDS},
  {"exc_set_object", exc_set_object, METH_VARARGS},
+ {"exc_set_object_fetch", exc_set_object_fetch, METH_VARARGS},
  {"raise_exception", raise_exception, METH_VARARGS},
  {"raise_memoryerror", raise_memoryerror, METH_NOARGS},
  {"set_exc_info", test_set_exc_info, METH_VARARGS},

Objects/exceptions.c

@@ -3749,6 +3749,21 @@ _PyExc_Fini(PyInterpreterState *interp)
  _PyExc_FiniTypes(interp);
 }
 
+int
+_PyException_AddNote(PyObject *exc, PyObject *note)
+{
+ if (!PyExceptionInstance_Check(exc)) {
+ PyErr_Format(PyExc_TypeError,
+ "exc must be an exception, not '%s'",
+ Py_TYPE(exc)->tp_name);
+ return -1;
+ }
+ PyObject *r = BaseException_add_note(exc, note);
+ int res = r == NULL ? -1 : 0;
+ Py_XDECREF(r);
+ return res;
+}
+
 /* Helper to do the equivalent of "raise X from Y" in C, but always using
  * the current exception rather than passing one in.
  *

Python/bltinmodule.c

@@ -3098,9 +3098,6 @@ _PyBuiltin_Init(PyInterpreterState *interp)
  }
  Py_DECREF(debug);
 
- /* m_copy of Py_None means it is copied some other way. */
- builtinsmodule.m_base.m_copy = Py_NewRef(Py_None);
-
  return mod;
 #undef ADD_TO_ALL
 #undef SETBUILTIN

Python/errors.c

@@ -135,6 +135,28 @@ _PyErr_GetTopmostException(PyThreadState *tstate)
  return exc_info;
 }
 
+static PyObject *
+get_normalization_failure_note(PyThreadState *tstate, PyObject *exception, PyObject *value)
+{
+ PyObject *args = PyObject_Repr(value);
+ if (args == NULL) {
+ _PyErr_Clear(tstate);
+ args = PyUnicode_FromFormat("<unknown>");
+ }
+ PyObject *note;
+ const char *tpname = ((PyTypeObject*)exception)->tp_name;
+ if (args == NULL) {
+ _PyErr_Clear(tstate);
+ note = PyUnicode_FromFormat("Normalization failed: type=%s", tpname);
+ }
+ else {
+ note = PyUnicode_FromFormat("Normalization failed: type=%s args=%S",
+ tpname, args);
+ Py_DECREF(args);
+ }
+ return note;
+}
+
 void
 _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value)
 {
@@ -160,19 +182,27 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value)
  Py_XINCREF(value);
  if (!is_subclass) {
  /* We must normalize the value right now */
- PyObject *fixed_value;
 
  /* Issue #23571: functions must not be called with an
  exception set */
  _PyErr_Clear(tstate);
 
- fixed_value = _PyErr_CreateException(exception, value);
- Py_XDECREF(value);
+ PyObject *fixed_value = _PyErr_CreateException(exception, value);
  if (fixed_value == NULL) {
+ PyObject *exc = _PyErr_GetRaisedException(tstate);
+ assert(PyExceptionInstance_Check(exc));
+
+ PyObject *note = get_normalization_failure_note(tstate, exception, value);
+ Py_XDECREF(value);
+ if (note != NULL) {
+ /* ignore errors in _PyException_AddNote - they will be overwritten below */
+ _PyException_AddNote(exc, note);
+ Py_DECREF(note);
+ }
+ _PyErr_SetRaisedException(tstate, exc);
  return;
  }
-
- value = fixed_value;
+ Py_XSETREF(value, fixed_value);
  }
 
  exc_value = _PyErr_GetTopmostException(tstate)->exc_value;