diff --git a/docs-requirements.in b/docs-requirements.in index c587c9b3f9..7d6abc0033 100644 --- a/docs-requirements.in +++ b/docs-requirements.in @@ -16,6 +16,7 @@ async_generator >= 1.9 idna outcome sniffio +exceptiongroup >= 1.0.0rc9 # See note in test-requirements.in immutables >= 0.6 diff --git a/docs-requirements.txt b/docs-requirements.txt index 7e30a9761f..5e022f3226 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # pip-compile docs-requirements.in @@ -28,6 +28,8 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme +exceptiongroup==1.0.0rc9 + # via -r docs-requirements.in idna==3.4 # via # -r docs-requirements.in @@ -36,6 +38,8 @@ imagesize==1.4.1 # via sphinx immutables==0.18 # via -r docs-requirements.in +importlib-metadata==4.12.0 + # via click incremental==21.3.0 # via towncrier jinja2==3.0.3 @@ -88,8 +92,14 @@ tomli==2.0.1 # via towncrier towncrier==22.8.0 # via -r docs-requirements.in +typing-extensions==4.3.0 + # via + # immutables + # importlib-metadata urllib3==1.26.12 # via requests +zipp==3.8.1 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/docs/source/conf.py b/docs/source/conf.py index 52872d0cb4..7859dd9e19 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -84,8 +84,9 @@ def setup(app): 'local_customization', ] +# FIXME: change the "python" link back to /3 when Python 3.11 is released intersphinx_mapping = { - "python": ('https://docs.python.org/3', None), + "python": ('https://docs.python.org/3.11', None), "outcome": ('https://outcome.readthedocs.io/en/latest/', None), "pyopenssl": ('https://www.pyopenssl.org/en/stable/', None), } diff --git a/docs/source/design.rst b/docs/source/design.rst index e0a8f939b7..c3a47ab30c 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -461,15 +461,9 @@ of our public APIs without having to modify Trio internals. Inside ``trio._core`` ~~~~~~~~~~~~~~~~~~~~~ -There are two notable sub-modules that are largely independent of -the rest of Trio, and could (possibly should?) be extracted into their -own independent packages: - -* ``_multierror.py``: Implements :class:`MultiError` and associated - infrastructure. - -* ``_ki.py``: Implements the core infrastructure for safe handling of - :class:`KeyboardInterrupt`. +The ``_ki.py`` module implements the core infrastructure for safe handling +of :class:`KeyboardInterrupt`. It's largely independent of the rest of Trio, +and could (possibly should?) be extracted into its own independent package. The most important submodule, where everything is integrated, is ``_run.py``. (This is also by far the largest submodule; it'd be nice diff --git a/docs/source/history.rst b/docs/source/history.rst index 414a5c518c..783fae01ba 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -48,8 +48,8 @@ Features Bugfixes ~~~~~~~~ -- Trio now avoids creating cyclic garbage when a `MultiError` is generated and filtered, - including invisibly within the cancellation system. This means errors raised +- Trio now avoids creating cyclic garbage when a ``MultiError`` is generated and + filtered, including invisibly within the cancellation system. This means errors raised through nurseries and cancel scopes should result in less GC latency. (`#2063 `__) - Trio now deterministically cleans up file descriptors that were opened before subprocess creation fails. Previously, they would remain open until the next run of @@ -288,9 +288,9 @@ Bugfixes - On Ubuntu systems, the system Python includes a custom unhandled-exception hook to perform `crash reporting `__. Unfortunately, Trio wants to use - the same hook to print nice `MultiError` tracebacks, causing a + the same hook to print nice ``MultiError`` tracebacks, causing a conflict. Previously, Trio would detect the conflict, print a warning, - and you just wouldn't get nice `MultiError` tracebacks. Now, Trio has + and you just wouldn't get nice ``MultiError`` tracebacks. Now, Trio has gotten clever enough to integrate its hook with Ubuntu's, so the two systems should Just Work together. (`#1065 `__) - Fixed an over-strict test that caused failures on Alpine Linux. @@ -492,7 +492,7 @@ Features violated. (One common source of such violations is an async generator that yields within a cancel scope.) The previous behavior was an inscrutable chain of TrioInternalErrors. (`#882 `__) -- MultiError now defines its ``exceptions`` attribute in ``__init__()`` +- ``MultiError`` now defines its ``exceptions`` attribute in ``__init__()`` to better support linters and code autocompletion. (`#1066 `__) - Use ``__slots__`` in more places internally, which should make Trio slightly faster. (`#984 `__) @@ -513,7 +513,7 @@ Bugfixes :meth:`~trio.Path.cwd`, are now async functions. Previously, a bug in the forwarding logic meant :meth:`~trio.Path.cwd` was synchronous and :meth:`~trio.Path.home` didn't work at all. (`#960 `__) -- An exception encapsulated within a :class:`MultiError` doesn't need to be +- An exception encapsulated within a ``MultiError`` doesn't need to be hashable anymore. .. note:: @@ -1304,7 +1304,7 @@ Other changes interfering with direct use of :func:`~trio.testing.wait_all_tasks_blocked` in the same test. -* :meth:`MultiError.catch` now correctly preserves ``__context__``, +* ``MultiError.catch()`` now correctly preserves ``__context__``, despite Python's best attempts to stop us (`#165 `__) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index e8d4bd6762..7fe2e04f6e 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -641,7 +641,7 @@ crucial things to keep in mind: * Any unhandled exceptions are re-raised inside the parent task. If there are multiple exceptions, then they're collected up into a - single :exc:`MultiError` exception. + single :exc:`BaseExceptionGroup` or :exc:`ExceptionGroup` exception. Since all tasks are descendents of the initial task, one consequence of this is that :func:`run` can't finish until all tasks have @@ -687,6 +687,8 @@ You might wonder why Trio can't just remember "this task should be cancelled in If you want a timeout to apply to one task but not another, then you need to put the cancel scope in that individual task's function -- ``child()``, in this example. +.. _exceptiongroups: + Errors in multiple child tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -709,18 +711,102 @@ limitation. Consider code like:: ``broken1`` raises ``KeyError``. ``broken2`` raises ``IndexError``. Obviously ``parent`` should raise some error, but -what? In some sense, the answer should be "both of these at once", but -in Python there can only be one exception at a time. +what? The answer is that both exceptions are grouped in an :exc:`ExceptionGroup`. +:exc:`ExceptionGroup` and its parent class :exc:`BaseExceptionGroup` are used to +encapsulate multiple exceptions being raised at once. + +To catch individual exceptions encapsulated in an exception group, the ``except*`` +clause was introduced in Python 3.11 (:pep:`654`). Here's how it works:: + + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(broken1) + nursery.start_soon(broken2) + except* KeyError as excgroup: + for exc in excgroup.exceptions: + ... # handle each KeyError + except* IndexError as excgroup: + for exc in excgroup.exceptions: + ... # handle each IndexError + +If you want to reraise exceptions, or raise new ones, you can do so, but be aware that +exceptions raised in ``except*`` sections will be raised together in a new exception +group. + +But what if you can't use ``except*`` just yet? Well, for that there is the handy +exceptiongroup_ library which lets you approximate this behavior with exception handler +callbacks:: + + from exceptiongroup import catch + + def handle_keyerrors(excgroup): + for exc in excgroup.exceptions: + ... # handle each KeyError + + def handle_indexerrors(excgroup): + for exc in excgroup.exceptions: + ... # handle each IndexError + + with catch({ + KeyError: handle_keyerror, + IndexError: handle_indexerror + }): + async with trio.open_nursery() as nursery: + nursery.start_soon(broken1) + nursery.start_soon(broken2) -Trio's answer is that it raises a :exc:`MultiError` object. This is a -special exception which encapsulates multiple exception objects – -either regular exceptions or nested :exc:`MultiError`\s. To make these -easier to work with, Trio installs a custom `sys.excepthook` that -knows how to print nice tracebacks for unhandled :exc:`MultiError`\s, -and it also provides some helpful utilities like -:meth:`MultiError.catch`, which allows you to catch "part of" a -:exc:`MultiError`. +The semantics for the handler functions are equal to ``except*`` blocks, except for +setting local variables. If you need to set local variables, you need to declare them +inside the handler function(s) with the ``nonlocal`` keyword:: + def handle_keyerrors(excgroup): + nonlocal myflag + myflag = True + + myflag = False + with catch({KeyError: handle_keyerror}): + async with trio.open_nursery() as nursery: + nursery.start_soon(broken1) + +For reasons of backwards compatibility, nurseries raise ``trio.MultiError`` and +``trio.NonBaseMultiError`` which inherit from :exc:`BaseExceptionGroup` and +:exc:`ExceptionGroup`, respectively. Users should refrain from attempting to raise or +catch the Trio specific exceptions themselves, and treat them as if they were standard +:exc:`BaseExceptionGroup` or :exc:`ExceptionGroup` instances instead. + +"Strict" versus "loose" ExceptionGroup semantics +++++++++++++++++++++++++++++++++++++++++++++++++ + +Ideally, in some abstract sense we'd want everything that *can* raise an +`ExceptionGroup` to *always* raise an `ExceptionGroup` (rather than, say, a single +`ValueError`). Otherwise, it would be easy to accidentally write something like ``except +ValueError:`` (not ``except*``), which works if a single exception is raised but fails to +catch _anything_ in the case of multiple simultaneous exceptions (even if one of them is +a ValueError). However, this is not how Trio worked in the past: as a concession to +practicality when the ``except*`` syntax hadn't been dreamed up yet, the old +``trio.MultiError`` was raised only when at least two exceptions occurred +simultaneously. Adding a layer of `ExceptionGroup` around every nursery, while +theoretically appealing, would probably break a lot of existing code in practice. + +Therefore, we've chosen to gate the newer, "stricter" behavior behind a parameter +called ``strict_exception_groups``. This is accepted as a parameter to +:func:`open_nursery`, to set the behavior for that nursery, and to :func:`trio.run`, +to set the default behavior for any nursery in your program that doesn't override it. + +* With ``strict_exception_groups=True``, the exception(s) coming out of a nursery will + always be wrapped in an `ExceptionGroup`, so you'll know that if you're handling + single errors correctly, multiple simultaneous errors will work as well. + +* With ``strict_exception_groups=False``, a nursery in which only one task has failed + will raise that task's exception without an additional layer of `ExceptionGroup` + wrapping, so you'll get maximum compatibility with code that was written to + support older versions of Trio. + +To maintain backwards compatibility, the default is ``strict_exception_groups=False``. +The default will eventually change to ``True`` in a future version of Trio, once +Python 3.11 and later versions are in wide use. + +.. _exceptiongroup: https://pypi.org/project/exceptiongroup/ Spawning tasks without becoming a parent ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -837,104 +923,6 @@ The nursery API See :meth:`~Nursery.start`. -Working with :exc:`MultiError`\s -++++++++++++++++++++++++++++++++ - -.. autoexception:: MultiError - - .. attribute:: exceptions - - The list of exception objects that this :exc:`MultiError` - represents. - - .. automethod:: filter - - .. automethod:: catch - :with: - -Examples: - -Suppose we have a handler function that discards :exc:`ValueError`\s:: - - def handle_ValueError(exc): - if isinstance(exc, ValueError): - return None - else: - return exc - -Then these both raise :exc:`KeyError`:: - - with MultiError.catch(handle_ValueError): - raise MultiError([KeyError(), ValueError()]) - - with MultiError.catch(handle_ValueError): - raise MultiError([ - ValueError(), - MultiError([KeyError(), ValueError()]), - ]) - -And both of these raise nothing at all:: - - with MultiError.catch(handle_ValueError): - raise MultiError([ValueError(), ValueError()]) - - with MultiError.catch(handle_ValueError): - raise MultiError([ - MultiError([ValueError(), ValueError()]), - ValueError(), - ]) - -You can also return a new or modified exception, for example:: - - def convert_ValueError_to_MyCustomError(exc): - if isinstance(exc, ValueError): - # Similar to 'raise MyCustomError from exc' - new_exc = MyCustomError(...) - new_exc.__cause__ = exc - return new_exc - else: - return exc - -In the example above, we set ``__cause__`` as a form of explicit -context chaining. :meth:`MultiError.filter` and -:meth:`MultiError.catch` also perform implicit exception chaining – if -you return a new exception object, then the new object's -``__context__`` attribute will automatically be set to the original -exception. - -We also monkey patch :class:`traceback.TracebackException` to be able -to handle formatting :exc:`MultiError`\s. This means that anything that -formats exception messages like :mod:`logging` will work out of the -box:: - - import logging - - logging.basicConfig() - - try: - raise MultiError([ValueError("foo"), KeyError("bar")]) - except: - logging.exception("Oh no!") - raise - -Will properly log the inner exceptions: - -.. code-block:: none - - ERROR:root:Oh no! - Traceback (most recent call last): - File "", line 2, in - trio.MultiError: ValueError('foo',), KeyError('bar',) - - Details of embedded exception 1: - - ValueError: foo - - Details of embedded exception 2: - - KeyError: 'bar' - - .. _task-local-storage: Task-local storage diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 08206569f5..19289ca991 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -13,14 +13,13 @@ Tutorial still probably read this, because Trio is different.) Trio turns Python into a concurrent language. It takes the core - async/await syntax introduced in 3.5, and uses it to add three + async/await syntax introduced in 3.5, and uses it to add two new pieces of semantics: - cancel scopes: a generic system for managing timeouts and cancellation - nurseries: which let your program do multiple things at the same time - - MultiErrors: for when multiple things go wrong at once Of course it also provides a complete suite of APIs for doing networking, file I/O, using worker threads, @@ -57,8 +56,6 @@ Tutorial and demonstrate start() then point out that you can just use serve_tcp() - exceptions and MultiError - example: catch-all logging in our echo server review of the three (or four) core language extensions @@ -1149,9 +1146,6 @@ TODO: explain :exc:`Cancelled` TODO: explain how cancellation is also used when one child raises an exception -TODO: show an example :exc:`MultiError` traceback and walk through its -structure - TODO: maybe a brief discussion of :exc:`KeyboardInterrupt` handling? .. diff --git a/docs/source/tutorial/echo-server.py b/docs/source/tutorial/echo-server.py index ffa63ab5f4..d37e509af4 100644 --- a/docs/source/tutorial/echo-server.py +++ b/docs/source/tutorial/echo-server.py @@ -21,18 +21,20 @@ async def echo_server(server_stream): print(f"echo_server {ident}: received data {data!r}") await server_stream.send_all(data) print(f"echo_server {ident}: connection closed") - # FIXME: add discussion of MultiErrors to the tutorial, and use - # MultiError.catch here. (Not important in this case, but important if the - # server code uses nurseries internally.) + # FIXME: add discussion of (Base)ExceptionGroup to the tutorial, and use + # exceptiongroup.catch() here. (Not important in this case, but important + # if the server code uses nurseries internally.) except Exception as exc: # Unhandled exceptions will propagate into our parent and take # down the whole program. If the exception is KeyboardInterrupt, # that's what we want, but otherwise maybe not... print(f"echo_server {ident}: crashed: {exc!r}") + async def main(): await trio.serve_tcp(echo_server, PORT) + # We could also just write 'trio.run(trio.serve_tcp, echo_server, PORT)', but real # programs almost always end up doing other stuff too and then we'd have to go # back and factor it out into a separate function anyway. So it's simplest to diff --git a/newsfragments/2211.headline.rst b/newsfragments/2211.headline.rst new file mode 100644 index 0000000000..c4320ba37f --- /dev/null +++ b/newsfragments/2211.headline.rst @@ -0,0 +1,13 @@ +``MultiError`` has been deprecated in favor of the standard :exc:`BaseExceptionGroup` +(introduced in :pep:`654`). On Python versions below 3.11, this exception and its +derivative :exc:`ExceptionGroup` are provided by the backport_. Trio still raises +``MultiError``, but it has been refactored into a subclass of :exc:`BaseExceptionGroup` +which users should catch instead of ``MultiError``. Uses of the ``MultiError.filter()`` +class method should be replaced with :meth:`BaseExceptionGroup.split`. Uses of the +``MultiError.catch()`` class method should be replaced with either ``except*`` clauses +(on Python 3.11+) or the ``exceptiongroup.catch()`` context manager provided by the +backport_. + +See the :ref:`updated documentation ` for details. + +.. _backport: https://pypi.org/project/exceptiongroup/ diff --git a/setup.py b/setup.py index 7021664c8d..00ba270764 100644 --- a/setup.py +++ b/setup.py @@ -89,6 +89,7 @@ # cffi 1.14 fixes memory leak inside ffi.getwinerror() # cffi is required on Windows, except on PyPy where it is built-in "cffi>=1.14; os_name == 'nt' and implementation_name != 'pypy'", + "exceptiongroup >= 1.0.0rc9; python_version < '3.11'", ], # This means, just install *everything* you see under trio/, even if it # doesn't look like a source file, so long as it appears in MANIFEST.in: diff --git a/test-requirements.in b/test-requirements.in index 186b13392e..cb9db8f894 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -30,3 +30,5 @@ async_generator >= 1.9 idna outcome sniffio +exceptiongroup >= 1.0.0rc9; python_version < "3.11" + diff --git a/test-requirements.txt b/test-requirements.txt index 8780ddca67..88123b0d69 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # pip-compile test-requirements.in @@ -34,12 +34,20 @@ decorator==5.1.1 # via ipython dill==0.3.5.1 # via pylint +exceptiongroup==1.0.0rc9 ; python_version < "3.11" + # via -r test-requirements.in flake8==4.0.1 # via -r test-requirements.in idna==3.4 # via # -r test-requirements.in # trustme +importlib-metadata==4.2.0 + # via + # click + # flake8 + # pluggy + # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 @@ -128,6 +136,12 @@ traitlets==5.4.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in +typed-ast==1.5.4 ; implementation_name == "cpython" and python_version < "3.8" + # via + # -r test-requirements.in + # astroid + # black + # mypy types-cryptography==3.3.22 # via types-pyopenssl types-pyopenssl==22.0.9 ; implementation_name == "cpython" @@ -135,11 +149,17 @@ types-pyopenssl==22.0.9 ; implementation_name == "cpython" typing-extensions==4.3.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black + # importlib-metadata # mypy + # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.14.1 # via astroid +zipp==3.8.1 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/trio/__init__.py b/trio/__init__.py index b35fa076b3..d6d2adb4bb 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -22,7 +22,6 @@ Cancelled, BusyResourceError, ClosedResourceError, - MultiError, run, open_nursery, CancelScope, @@ -90,6 +89,9 @@ serve_ssl_over_tcp, ) +from ._core._multierror import MultiError as _MultiError +from ._core._multierror import NonBaseMultiError as _NonBaseMultiError + from ._deprecate import TrioDeprecationWarning # Submodules imported by default @@ -115,6 +117,24 @@ issue=1104, instead="trio.lowlevel.open_process", ), + "MultiError": _deprecate.DeprecatedAttribute( + value=_MultiError, + version="0.22.0", + issue=2211, + instead=( + "BaseExceptionGroup (on Python 3.11 and later) or " + "exceptiongroup.BaseExceptionGroup (earlier versions)" + ), + ), + "NonBaseMultiError": _deprecate.DeprecatedAttribute( + value=_NonBaseMultiError, + version="0.22.0", + issue=2211, + instead=( + "ExceptionGroup (on Python 3.11 and later) or " + "exceptiongroup.ExceptionGroup (earlier versions)" + ), + ), } # Having the public path in .__module__ attributes is important for: diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index 2bd0c74e67..8e3e526cfe 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -17,8 +17,6 @@ EndOfChannel, ) -from ._multierror import MultiError - from ._ki import ( enable_ki_protection, disable_ki_protection, diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index de0d56d464..c95f05b4ca 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -1,10 +1,16 @@ import sys -import traceback -import textwrap import warnings +from typing import Sequence import attr +from trio._deprecate import warn_deprecated + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup, ExceptionGroup, print_exception +else: + from traceback import print_exception + ################################################################ # MultiError ################################################################ @@ -127,7 +133,7 @@ def __enter__(self): def __exit__(self, etype, exc, tb): if exc is not None: - filtered_exc = MultiError.filter(self._handler, exc) + filtered_exc = _filter_impl(self._handler, exc) if filtered_exc is exc: # Let the interpreter re-raise it @@ -151,7 +157,7 @@ def __exit__(self, etype, exc, tb): del _, filtered_exc, value -class MultiError(BaseException): +class MultiError(BaseExceptionGroup): """An exception that contains other exceptions; also known as an "inception". @@ -174,21 +180,23 @@ class MultiError(BaseException): """ - def __init__(self, exceptions): - # Avoid recursion when exceptions[0] returned by __new__() happens - # to be a MultiError and subsequently __init__() is called. - if hasattr(self, "exceptions"): - # __init__ was already called on this object - assert len(exceptions) == 1 and exceptions[0] is self + def __init__(self, exceptions, *, _collapse=True): + self.collapse = _collapse + + # Avoid double initialization when _collapse is True and exceptions[0] returned + # by __new__() happens to be a MultiError and subsequently __init__() is called. + if _collapse and getattr(self, "exceptions", None) is not None: + # This exception was already initialized. return - self.exceptions = exceptions - def __new__(cls, exceptions): + super().__init__("multiple tasks failed", exceptions) + + def __new__(cls, exceptions, *, _collapse=True): exceptions = list(exceptions) for exc in exceptions: if not isinstance(exc, BaseException): - raise TypeError("Expected an exception object, not {!r}".format(exc)) - if len(exceptions) == 1: + raise TypeError(f"Expected an exception object, not {exc!r}") + if _collapse and len(exceptions) == 1: # If this lone object happens to itself be a MultiError, then # Python will implicitly call our __init__ on it again. See # special handling in __init__. @@ -200,7 +208,10 @@ def __new__(cls, exceptions): # In an earlier version of the code, we didn't define __init__ and # simply set the `exceptions` attribute directly on the new object. # However, linters expect attributes to be initialized in __init__. - return BaseException.__new__(cls, exceptions) + if all(isinstance(exc, Exception) for exc in exceptions): + cls = NonBaseMultiError + + return super().__new__(cls, "multiple tasks failed", exceptions) def __str__(self): return ", ".join(repr(exc) for exc in self.exceptions) @@ -208,6 +219,13 @@ def __str__(self): def __repr__(self): return "".format(self) + def derive(self, __excs): + # We use _collapse=False here to get ExceptionGroup semantics, since derive() + # is part of the PEP 654 API + exc = MultiError(__excs, _collapse=False) + exc.collapse = self.collapse + return exc + @classmethod def filter(cls, handler, root_exc): """Apply the given ``handler`` to all the exceptions in ``root_exc``. @@ -224,7 +242,12 @@ def filter(cls, handler, root_exc): ``handler`` returned None for all the inputs, returns None. """ - + warn_deprecated( + "MultiError.filter()", + "0.22.0", + instead="BaseExceptionGroup.split()", + issue=2211, + ) return _filter_impl(handler, root_exc) @classmethod @@ -236,12 +259,23 @@ def catch(cls, handler): handler: as for :meth:`filter` """ + warn_deprecated( + "MultiError.catch", + "0.22.0", + instead="except* or exceptiongroup.catch()", + issue=2211, + ) return MultiErrorCatcher(handler) +class NonBaseMultiError(MultiError, ExceptionGroup): + pass + + # Clean up exception printing: MultiError.__module__ = "trio" +NonBaseMultiError.__module__ = "trio" ################################################################ # concat_tb @@ -361,95 +395,8 @@ def concat_tb(head, tail): return current_head -################################################################ -# MultiError traceback formatting -# -# What follows is terrible, terrible monkey patching of -# traceback.TracebackException to add support for handling -# MultiErrors -################################################################ - -traceback_exception_original_init = traceback.TracebackException.__init__ - - -def traceback_exception_init( - self, - exc_type, - exc_value, - exc_traceback, - *, - limit=None, - lookup_lines=True, - capture_locals=False, - compact=False, - _seen=None, - **kwargs, -): - if sys.version_info >= (3, 10): - kwargs["compact"] = compact - - # Capture the original exception and its cause and context as TracebackExceptions - traceback_exception_original_init( - self, - exc_type, - exc_value, - exc_traceback, - limit=limit, - lookup_lines=lookup_lines, - capture_locals=capture_locals, - _seen=_seen, - **kwargs, - ) - - seen_was_none = _seen is None - - if _seen is None: - _seen = set() - - # Capture each of the exceptions in the MultiError along with each of their causes and contexts - if isinstance(exc_value, MultiError): - embedded = [] - for exc in exc_value.exceptions: - if id(exc) not in _seen: - embedded.append( - traceback.TracebackException.from_exception( - exc, - limit=limit, - lookup_lines=lookup_lines, - capture_locals=capture_locals, - # copy the set of _seen exceptions so that duplicates - # shared between sub-exceptions are not omitted - _seen=None if seen_was_none else set(_seen), - **kwargs, - ) - ) - self.embedded = embedded - else: - self.embedded = [] - - -traceback.TracebackException.__init__ = traceback_exception_init # type: ignore -traceback_exception_original_format = traceback.TracebackException.format - - -def traceback_exception_format(self, *, chain=True): - yield from traceback_exception_original_format(self, chain=chain) - - for i, exc in enumerate(self.embedded): - yield "\nDetails of embedded exception {}:\n\n".format(i + 1) - yield from (textwrap.indent(line, " " * 2) for line in exc.format(chain=chain)) - - -traceback.TracebackException.format = traceback_exception_format # type: ignore - - -def trio_excepthook(etype, value, tb): - for chunk in traceback.format_exception(etype, value, tb): - sys.stderr.write(chunk) - - -monkeypatched_or_warned = False - +# Remove when IPython gains support for exception groups +# (https://github.com/ipython/ipython/issues/13753) if "IPython" in sys.modules: import IPython @@ -459,24 +406,19 @@ def trio_excepthook(etype, value, tb): warnings.warn( "IPython detected, but you already have a custom exception " "handler installed. I'll skip installing Trio's custom " - "handler, but this means MultiErrors will not show full " + "handler, but this means exception groups will not show full " "tracebacks.", category=RuntimeWarning, ) - monkeypatched_or_warned = True else: def trio_show_traceback(self, etype, value, tb, tb_offset=None): # XX it would be better to integrate with IPython's fancy # exception formatting stuff (and not ignore tb_offset) - trio_excepthook(etype, value, tb) + print_exception(value) - ip.set_custom_exc((MultiError,), trio_show_traceback) - monkeypatched_or_warned = True + ip.set_custom_exc((BaseExceptionGroup,), trio_show_traceback) -if sys.excepthook is sys.__excepthook__: - sys.excepthook = trio_excepthook - monkeypatched_or_warned = True # Ubuntu's system Python has a sitecustomize.py file that import # apport_python_hook and replaces sys.excepthook. @@ -490,27 +432,21 @@ def trio_show_traceback(self, etype, value, tb, tb_offset=None): # hook. # # More details: https://github.com/python-trio/trio/issues/1065 -if getattr(sys.excepthook, "__name__", None) == "apport_excepthook": +if ( + sys.version_info < (3, 11) + and getattr(sys.excepthook, "__name__", None) == "apport_excepthook" +): + from types import ModuleType + import apport_python_hook + from exceptiongroup import format_exception assert sys.excepthook is apport_python_hook.apport_excepthook - # Give it a descriptive name as a hint for anyone who's stuck trying to - # debug this mess later. - class TrioFakeSysModuleForApport: - pass + def replacement_excepthook(etype, value, tb): + sys.stderr.write("".join(format_exception(etype, value, tb))) - fake_sys = TrioFakeSysModuleForApport() + fake_sys = ModuleType("trio_fake_sys") fake_sys.__dict__.update(sys.__dict__) - fake_sys.__excepthook__ = trio_excepthook # type: ignore + fake_sys.__excepthook__ = replacement_excepthook # type: ignore apport_python_hook.sys = fake_sys - - monkeypatched_or_warned = True - -if not monkeypatched_or_warned: - warnings.warn( - "You seem to already have a custom sys.excepthook handler " - "installed. I'll skip installing Trio's custom handler, but this " - "means MultiErrors will not show full tracebacks.", - category=RuntimeWarning, - ) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 1eb957b8d8..7e57cfa445 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -28,7 +28,7 @@ KIManager, enable_ki_protection, ) -from ._multierror import MultiError +from ._multierror import MultiError, concat_tb from ._traps import ( Abort, wait_task_rescheduled, @@ -43,6 +43,9 @@ from .. import _core from .._util import Final, NoPublicConstructor, coroutine_or_error +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + DEADLINE_HEAP_MIN_PRUNE_THRESHOLD = 1000 _NO_SEND = object() @@ -116,6 +119,31 @@ class IdlePrimedTypes(enum.Enum): ################################################################ +def collapse_exception_group(excgroup): + """Recursively collapse any single-exception groups into that single contained + exception. + + """ + exceptions = list(excgroup.exceptions) + modified = False + for i, exc in enumerate(exceptions): + if isinstance(exc, BaseExceptionGroup): + new_exc = collapse_exception_group(exc) + if new_exc is not exc: + modified = True + exceptions[i] = new_exc + + if len(exceptions) == 1 and isinstance(excgroup, MultiError) and excgroup.collapse: + exceptions[0].__traceback__ = concat_tb( + excgroup.__traceback__, exceptions[0].__traceback__ + ) + return exceptions[0] + elif modified: + return excgroup.derive(exceptions) + else: + return excgroup + + @attr.s(eq=False, slots=True) class Deadlines: """A container of deadlined cancel scopes. @@ -447,12 +475,6 @@ def __enter__(self): task._activate_cancel_status(self._cancel_status) return self - def _exc_filter(self, exc): - if isinstance(exc, Cancelled): - self.cancelled_caught = True - return None - return exc - def _close(self, exc): if self._cancel_status is None: new_exc = RuntimeError( @@ -510,7 +532,17 @@ def _close(self, exc): and self._cancel_status.effectively_cancelled and not self._cancel_status.parent_cancellation_is_visible_to_us ): - exc = MultiError.filter(self._exc_filter, exc) + if isinstance(exc, Cancelled): + self.cancelled_caught = True + exc = None + elif isinstance(exc, BaseExceptionGroup): + matched, exc = exc.split(Cancelled) + if matched: + self.cancelled_caught = True + + if exc: + exc = collapse_exception_group(exc) + self._cancel_status.close() with self._might_change_registered_deadline(): self._cancel_status = None @@ -778,6 +810,7 @@ def started(self, value=None): self._old_nursery._check_nursery_closed() +@attr.s class NurseryManager: """Nursery context manager. @@ -788,11 +821,15 @@ class NurseryManager: """ + strict_exception_groups = attr.ib(default=False) + @enable_ki_protection async def __aenter__(self): self._scope = CancelScope() self._scope.__enter__() - self._nursery = Nursery._create(current_task(), self._scope) + self._nursery = Nursery._create( + current_task(), self._scope, self.strict_exception_groups + ) return self._nursery @enable_ki_protection @@ -828,15 +865,23 @@ def __exit__(self): # pragma: no cover assert False, """Never called, but should be defined""" -def open_nursery(): +def open_nursery(strict_exception_groups=None): """Returns an async context manager which must be used to create a new `Nursery`. It does not block on entry; on exit it blocks until all child tasks have exited. + Args: + strict_exception_groups (bool): If true, even a single raised exception will be + wrapped in an exception group. This will eventually become the default + behavior. If not specified, uses the value passed to :func:`run`. + """ - return NurseryManager() + if strict_exception_groups is None: + strict_exception_groups = GLOBAL_RUN_CONTEXT.runner.strict_exception_groups + + return NurseryManager(strict_exception_groups=strict_exception_groups) class Nursery(metaclass=NoPublicConstructor): @@ -861,8 +906,9 @@ class Nursery(metaclass=NoPublicConstructor): in response to some external event. """ - def __init__(self, parent_task, cancel_scope): + def __init__(self, parent_task, cancel_scope, strict_exception_groups): self._parent_task = parent_task + self._strict_exception_groups = strict_exception_groups parent_task._child_nurseries.append(self) # the cancel status that children inherit - we take a snapshot, so it # won't be affected by any changes in the parent. @@ -910,7 +956,8 @@ def _child_finished(self, task, outcome): self._check_nursery_closed() async def _nested_child_finished(self, nested_child_exc): - """Returns MultiError instance if there are pending exceptions.""" + # Returns MultiError instance (or any exception if the nursery is in loose mode + # and there is just one contained exception) if there are pending exceptions if nested_child_exc is not None: self._add_exc(nested_child_exc) self._nested_child_running = False @@ -939,7 +986,9 @@ def aborted(raise_cancel): assert popped is self if self._pending_excs: try: - return MultiError(self._pending_excs) + return MultiError( + self._pending_excs, _collapse=not self._strict_exception_groups + ) finally: # avoid a garbage cycle # (see test_nursery_cancel_doesnt_create_cyclic_garbage) @@ -1278,6 +1327,7 @@ class Runner: instruments: Instruments = attr.ib() io_manager = attr.ib() ki_manager = attr.ib() + strict_exception_groups = attr.ib() # Run-local values, see _local.py _locals = attr.ib(factory=dict) @@ -1816,7 +1866,12 @@ def abort(_): # 'is_guest' to see the special cases we need to handle this. -def setup_runner(clock, instruments, restrict_keyboard_interrupt_to_checkpoints): +def setup_runner( + clock, + instruments, + restrict_keyboard_interrupt_to_checkpoints, + strict_exception_groups, +): """Create a Runner object and install it as the GLOBAL_RUN_CONTEXT.""" # It wouldn't be *hard* to support nested calls to run(), but I can't # think of a single good reason for it, so let's be conservative for @@ -1838,6 +1893,7 @@ def setup_runner(clock, instruments, restrict_keyboard_interrupt_to_checkpoints) io_manager=io_manager, system_context=system_context, ki_manager=ki_manager, + strict_exception_groups=strict_exception_groups, ) runner.asyncgens.install_hooks(runner) @@ -1855,6 +1911,7 @@ def run( clock=None, instruments=(), restrict_keyboard_interrupt_to_checkpoints=False, + strict_exception_groups=False, ): """Run a Trio-flavored async function, and return the result. @@ -1911,6 +1968,10 @@ def run( main thread (this is a Python limitation), or if you use :func:`open_signal_receiver` to catch SIGINT. + strict_exception_groups (bool): If true, nurseries will always wrap even a single + raised exception in an exception group. This can be overridden on the level of + individual nurseries. This will eventually become the default behavior. + Returns: Whatever ``async_fn`` returns. @@ -1927,7 +1988,10 @@ def run( __tracebackhide__ = True runner = setup_runner( - clock, instruments, restrict_keyboard_interrupt_to_checkpoints + clock, + instruments, + restrict_keyboard_interrupt_to_checkpoints, + strict_exception_groups, ) gen = unrolled_run(runner, async_fn, args) @@ -1956,6 +2020,7 @@ def start_guest_run( clock=None, instruments=(), restrict_keyboard_interrupt_to_checkpoints=False, + strict_exception_groups=False, ): """Start a "guest" run of Trio on top of some other "host" event loop. @@ -2007,7 +2072,10 @@ def my_done_callback(run_outcome): """ runner = setup_runner( - clock, instruments, restrict_keyboard_interrupt_to_checkpoints + clock, + instruments, + restrict_keyboard_interrupt_to_checkpoints, + strict_exception_groups, ) runner.is_guest = True runner.guest_tick_scheduled = True diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index e4a30e0e69..edebf0dd59 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -1,5 +1,9 @@ import gc import logging +import os +import subprocess +from pathlib import Path + import pytest from traceback import ( @@ -9,16 +13,16 @@ ) from traceback import _cause_message # type: ignore import sys -import os import re -from pathlib import Path -import subprocess from .tutil import slow - -from .._multierror import MultiError, concat_tb +from .._multierror import MultiError, concat_tb, NonBaseMultiError +from ... import TrioDeprecationWarning from ..._core import open_nursery +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + class NotHashableException(Exception): code = None @@ -72,10 +76,6 @@ def get_tb(raiser): return get_exc(raiser).__traceback__ -def einfo(exc): - return (type(exc), exc, exc.__traceback__) - - def test_concat_tb(): tb1 = get_tb(raiser1) @@ -108,7 +108,7 @@ def test_MultiError(): assert MultiError([exc1]) is exc1 m = MultiError([exc1, exc2]) - assert m.exceptions == [exc1, exc2] + assert m.exceptions == (exc1, exc2) assert "ValueError" in str(m) assert "ValueError" in repr(m) @@ -121,7 +121,7 @@ def test_MultiError(): def test_MultiErrorOfSingleMultiError(): # For MultiError([MultiError]), ensure there is no bad recursion by the # constructor where __init__ is called if __new__ returns a bare MultiError. - exceptions = [KeyError(), ValueError()] + exceptions = (KeyError(), ValueError()) a = MultiError(exceptions) b = MultiError([a]) assert b == a @@ -150,19 +150,10 @@ def handle_ValueError(exc): else: return exc - filtered_excs = MultiError.filter(handle_ValueError, excs) - assert isinstance(filtered_excs, NotHashableException) + with pytest.warns(TrioDeprecationWarning): + filtered_excs = MultiError.filter(handle_ValueError, excs) - -def test_traceback_recursion(): - exc1 = RuntimeError() - exc2 = KeyError() - exc3 = NotHashableException(42) - # Note how this creates a loop, where exc1 refers to exc1 - # This could trigger an infinite recursion; the 'seen' set is supposed to prevent - # this. - exc1.__cause__ = MultiError([exc1, exc2, exc3]) - format_exception(*einfo(exc1)) + assert isinstance(filtered_excs, NotHashableException) def make_tree(): @@ -206,7 +197,9 @@ def null_handler(exc): m = make_tree() assert_tree_eq(m, m) - assert MultiError.filter(null_handler, m) is m + with pytest.warns(TrioDeprecationWarning): + assert MultiError.filter(null_handler, m) is m + assert_tree_eq(m, make_tree()) # Make sure we don't pick up any detritus if run in a context where @@ -215,7 +208,8 @@ def null_handler(exc): try: raise ValueError except ValueError: - assert MultiError.filter(null_handler, m) is m + with pytest.warns(TrioDeprecationWarning): + assert MultiError.filter(null_handler, m) is m assert_tree_eq(m, make_tree()) def simple_filter(exc): @@ -225,7 +219,9 @@ def simple_filter(exc): return RuntimeError() return exc - new_m = MultiError.filter(simple_filter, make_tree()) + with pytest.warns(TrioDeprecationWarning): + new_m = MultiError.filter(simple_filter, make_tree()) + assert isinstance(new_m, MultiError) assert len(new_m.exceptions) == 2 # was: [[ValueError, KeyError], NameError] @@ -267,7 +263,8 @@ def filter_NameError(exc): return exc m = make_tree() - new_m = MultiError.filter(filter_NameError, m) + with pytest.warns(TrioDeprecationWarning): + new_m = MultiError.filter(filter_NameError, m) # with the NameError gone, the other branch gets promoted assert new_m is m.exceptions[0] @@ -275,7 +272,8 @@ def filter_NameError(exc): def filter_all(exc): return None - assert MultiError.filter(filter_all, make_tree()) is None + with pytest.warns(TrioDeprecationWarning): + assert MultiError.filter(filter_all, make_tree()) is None def test_MultiError_catch(): @@ -284,13 +282,13 @@ def test_MultiError_catch(): def noop(_): pass # pragma: no cover - with MultiError.catch(noop): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(noop): pass # Simple pass-through of all exceptions m = make_tree() with pytest.raises(MultiError) as excinfo: - with MultiError.catch(lambda exc: exc): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(lambda exc: exc): raise m assert excinfo.value is m # Should be unchanged, except that we added a traceback frame by raising @@ -302,7 +300,7 @@ def noop(_): assert_tree_eq(m, make_tree()) # Swallows everything - with MultiError.catch(lambda _: None): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(lambda _: None): raise make_tree() def simple_filter(exc): @@ -313,7 +311,7 @@ def simple_filter(exc): return exc with pytest.raises(MultiError) as excinfo: - with MultiError.catch(simple_filter): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(simple_filter): raise make_tree() new_m = excinfo.value assert isinstance(new_m, MultiError) @@ -331,7 +329,7 @@ def simple_filter(exc): v = ValueError() v.__cause__ = KeyError() with pytest.raises(ValueError) as excinfo: - with MultiError.catch(lambda exc: exc): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(lambda exc: exc): raise v assert isinstance(excinfo.value.__cause__, KeyError) @@ -339,7 +337,7 @@ def simple_filter(exc): context = KeyError() v.__context__ = context with pytest.raises(ValueError) as excinfo: - with MultiError.catch(lambda exc: exc): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(lambda exc: exc): raise v assert excinfo.value.__context__ is context assert not excinfo.value.__suppress_context__ @@ -358,8 +356,9 @@ def catch_RuntimeError(exc): else: return exc - with MultiError.catch(catch_RuntimeError): - raise MultiError([v, distractor]) + with pytest.warns(TrioDeprecationWarning): + with MultiError.catch(catch_RuntimeError): + raise MultiError([v, distractor]) assert excinfo.value.__context__ is context assert excinfo.value.__suppress_context__ == suppress_context @@ -387,7 +386,7 @@ def simple_filter(exc): gc.set_debug(gc.DEBUG_SAVEALL) with pytest.raises(MultiError): # covers MultiErrorCatcher.__exit__ and _multierror.copy_tb - with MultiError.catch(simple_filter): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(simple_filter): raise make_multi() gc.collect() assert not gc.garbage @@ -414,199 +413,25 @@ def test_assert_match_in_seq(): assert_match_in_seq(["a", "b"], "xx b xx a xx") -def test_format_exception(): - exc = get_exc(raiser1) - formatted = "".join(format_exception(*einfo(exc))) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" not in formatted - assert "in raiser2_2" not in formatted - assert "direct cause" not in formatted - assert "During handling" not in formatted - - exc = get_exc(raiser1) - exc.__cause__ = get_exc(raiser2) - formatted = "".join(format_exception(*einfo(exc))) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" in formatted - assert "in raiser2_2" in formatted - assert "direct cause" in formatted - assert "During handling" not in formatted - # ensure cause included - assert _cause_message in formatted - - exc = get_exc(raiser1) - exc.__context__ = get_exc(raiser2) - formatted = "".join(format_exception(*einfo(exc))) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" in formatted - assert "in raiser2_2" in formatted - assert "direct cause" not in formatted - assert "During handling" in formatted - - exc.__suppress_context__ = True - formatted = "".join(format_exception(*einfo(exc))) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" not in formatted - assert "in raiser2_2" not in formatted - assert "direct cause" not in formatted - assert "During handling" not in formatted - - # chain=False - exc = get_exc(raiser1) - exc.__context__ = get_exc(raiser2) - formatted = "".join(format_exception(*einfo(exc), chain=False)) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" not in formatted - assert "in raiser2_2" not in formatted - assert "direct cause" not in formatted - assert "During handling" not in formatted - - # limit - exc = get_exc(raiser1) - exc.__context__ = get_exc(raiser2) - # get_exc adds a frame that counts against the limit, so limit=2 means we - # get 1 deep into the raiser stack - formatted = "".join(format_exception(*einfo(exc), limit=2)) - print(formatted) - assert "raiser1_string" in formatted - assert "in raiser1" in formatted - assert "in raiser1_2" not in formatted - assert "raiser2_string" in formatted - assert "in raiser2" in formatted - assert "in raiser2_2" not in formatted - - exc = get_exc(raiser1) - exc.__context__ = get_exc(raiser2) - formatted = "".join(format_exception(*einfo(exc), limit=1)) - print(formatted) - assert "raiser1_string" in formatted - assert "in raiser1" not in formatted - assert "raiser2_string" in formatted - assert "in raiser2" not in formatted - - # handles loops - exc = get_exc(raiser1) - exc.__cause__ = exc - formatted = "".join(format_exception(*einfo(exc))) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" not in formatted - assert "in raiser2_2" not in formatted - # ensure duplicate exception is not included as cause - assert _cause_message not in formatted - - # MultiError - formatted = "".join(format_exception(*einfo(make_tree()))) - print(formatted) - - assert_match_in_seq( - [ - # Outer exception is MultiError - r"MultiError:", - # First embedded exception is the embedded MultiError - r"\nDetails of embedded exception 1", - # Which has a single stack frame from make_tree raising it - r"in make_tree", - # Then it has two embedded exceptions - r" Details of embedded exception 1", - r"in raiser1_2", - # for some reason ValueError has no quotes - r"ValueError: raiser1_string", - r" Details of embedded exception 2", - r"in raiser2_2", - # But KeyError does have quotes - r"KeyError: 'raiser2_string'", - # And finally the NameError, which is a sibling of the embedded - # MultiError - r"\nDetails of embedded exception 2:", - r"in raiser3", - r"NameError", - ], - formatted, - ) - - # Prints duplicate exceptions in sub-exceptions - exc1 = get_exc(raiser1) - - def raise1_raiser1(): - try: - raise exc1 - except: - raise ValueError("foo") - - def raise2_raiser1(): - try: - raise exc1 - except: - raise KeyError("bar") - - exc2 = get_exc(raise1_raiser1) - exc3 = get_exc(raise2_raiser1) - - try: - raise MultiError([exc2, exc3]) - except MultiError as e: - exc = e +def test_base_multierror(): + """ + Test that MultiError() with at least one base exception will return a MultiError + object. + """ - formatted = "".join(format_exception(*einfo(exc))) - print(formatted) + exc = MultiError([ZeroDivisionError(), KeyboardInterrupt()]) + assert type(exc) is MultiError - assert_match_in_seq( - [ - r"Traceback", - # Outer exception is MultiError - r"MultiError:", - # First embedded exception is the embedded ValueError with cause of raiser1 - r"\nDetails of embedded exception 1", - # Print details of exc1 - r" Traceback", - r"in get_exc", - r"in raiser1", - r"ValueError: raiser1_string", - # Print details of exc2 - r"\n During handling of the above exception, another exception occurred:", - r" Traceback", - r"in get_exc", - r"in raise1_raiser1", - r" ValueError: foo", - # Second embedded exception is the embedded KeyError with cause of raiser1 - r"\nDetails of embedded exception 2", - # Print details of exc1 again - r" Traceback", - r"in get_exc", - r"in raiser1", - r"ValueError: raiser1_string", - # Print details of exc3 - r"\n During handling of the above exception, another exception occurred:", - r" Traceback", - r"in get_exc", - r"in raise2_raiser1", - r" KeyError: 'bar'", - ], - formatted, - ) +def test_non_base_multierror(): + """ + Test that MultiError() without base exceptions will return a NonBaseMultiError + object. + """ -def test_logging(caplog): - exc1 = get_exc(raiser1) - exc2 = get_exc(raiser2) - - m = MultiError([exc1, exc2]) - - message = "test test test" - try: - raise m - except MultiError as exc: - logging.getLogger().exception(message) - # Join lines together - formatted = "".join(format_exception(type(exc), exc, exc.__traceback__)) - assert message in caplog.text - assert formatted in caplog.text + exc = MultiError([ZeroDivisionError(), ValueError()]) + assert type(exc) is NonBaseMultiError + assert isinstance(exc, ExceptionGroup) def run_script(name, use_ipython=False): @@ -654,10 +479,10 @@ def check_simple_excepthook(completed): [ "in ", "MultiError", - "Details of embedded exception 1", + "--- 1 ---", "in exc1_fn", "ValueError", - "Details of embedded exception 2", + "--- 2 ---", "in exc2_fn", "KeyError", ], @@ -665,38 +490,6 @@ def check_simple_excepthook(completed): ) -def test_simple_excepthook(): - completed = run_script("simple_excepthook.py") - check_simple_excepthook(completed) - - -def test_custom_excepthook(): - # Check that user-defined excepthooks aren't overridden - completed = run_script("custom_excepthook.py") - assert_match_in_seq( - [ - # The warning - "RuntimeWarning", - "already have a custom", - # The message printed by the custom hook, proving we didn't - # override it - "custom running!", - # The MultiError - "MultiError:", - ], - completed.stdout.decode("utf-8"), - ) - - -# This warning is triggered by ipython 7.5.0 on python 3.8 -import warnings - -warnings.filterwarnings( - "ignore", - message='.*"@coroutine" decorator is deprecated', - category=DeprecationWarning, - module="IPython.*", -) try: import IPython except ImportError: # pragma: no cover @@ -721,15 +514,6 @@ def test_ipython_imported_but_unused(): check_simple_excepthook(completed) -@slow -def test_partial_imported_but_unused(): - # Check that a functools.partial as sys.excepthook doesn't cause an exception when - # importing trio. This was a problem due to the lack of a .__name__ attribute and - # happens when inside a pytest-qt test case for example. - completed = run_script("simple_excepthook_partial.py") - completed.check_returncode() - - @slow @need_ipython def test_ipython_custom_exc_handler(): @@ -767,6 +551,6 @@ def test_apport_excepthook_monkeypatch_interaction(): # Proper traceback assert_match_in_seq( - ["Details of embedded", "KeyError", "Details of embedded", "ValueError"], + ["--- 1 ---", "KeyError", "--- 2 ---", "ValueError"], stdout, ) diff --git a/trio/_core/tests/test_multierror_scripts/custom_excepthook.py b/trio/_core/tests/test_multierror_scripts/custom_excepthook.py deleted file mode 100644 index 564c5833b2..0000000000 --- a/trio/_core/tests/test_multierror_scripts/custom_excepthook.py +++ /dev/null @@ -1,18 +0,0 @@ -import _common - -import sys - - -def custom_excepthook(*args): - print("custom running!") - return sys.__excepthook__(*args) - - -sys.excepthook = custom_excepthook - -# Should warn that we'll get kinda-broken tracebacks -import trio - -# The custom excepthook should run, because Trio was polite and didn't -# override it -raise trio.MultiError([ValueError(), KeyError()]) diff --git a/trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py b/trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py deleted file mode 100644 index e97fc39d57..0000000000 --- a/trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py +++ /dev/null @@ -1,13 +0,0 @@ -import functools -import sys - -import _common - -# just making sure importing Trio doesn't fail if sys.excepthook doesn't have a -# .__name__ attribute - -sys.excepthook = functools.partial(sys.excepthook) - -assert not hasattr(sys.excepthook, "__name__") - -import trio diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 863dad9909..262ad6054b 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -28,6 +28,7 @@ ) from ... import _core +from ..._core._multierror import MultiError, NonBaseMultiError from .._run import DEADLINE_HEAP_MIN_PRUNE_THRESHOLD from ..._threads import to_thread_run_sync from ..._timeouts import sleep, fail_after @@ -37,6 +38,9 @@ assert_checkpoints, ) +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + # slightly different from _timeouts.sleep_forever because it returns the value # its rescheduled with, which is really only useful for tests of @@ -177,7 +181,7 @@ async def main(): nursery.start_soon(crasher) raise KeyError - with pytest.raises(_core.MultiError) as excinfo: + with pytest.raises(MultiError) as excinfo: _core.run(main) print(excinfo.value) assert {type(exc) for exc in excinfo.value.exceptions} == { @@ -195,7 +199,7 @@ async def main(): nursery.start_soon(crasher, KeyError) nursery.start_soon(crasher, ValueError) - with pytest.raises(_core.MultiError) as excinfo: + with pytest.raises(MultiError) as excinfo: _core.run(main) assert {type(exc) for exc in excinfo.value.exceptions} == { ValueError, @@ -432,7 +436,7 @@ async def crasher(): # And one that raises a different error nursery.start_soon(crasher) # t4 # and then our __aexit__ also receives an outer Cancelled - except _core.MultiError as multi_exc: + except MultiError as multi_exc: # Since the outer scope became cancelled before the # nursery block exited, all cancellations inside the # nursery block continue propagating to reach the @@ -771,7 +775,7 @@ async def task2(): with pytest.raises(RuntimeError) as exc_info: await nursery_mgr.__aexit__(*sys.exc_info()) assert "which had already been exited" in str(exc_info.value) - assert type(exc_info.value.__context__) is _core.MultiError + assert type(exc_info.value.__context__) is NonBaseMultiError assert len(exc_info.value.__context__.exceptions) == 3 cancelled_in_context = False for exc in exc_info.value.__context__.exceptions: @@ -932,7 +936,7 @@ async def main(): _core.run(main) me = excinfo.value.__cause__ - assert isinstance(me, _core.MultiError) + assert isinstance(me, MultiError) assert len(me.exceptions) == 2 for exc in me.exceptions: assert isinstance(exc, (KeyError, ValueError)) @@ -1111,7 +1115,7 @@ async def test_nursery_exception_chaining_doesnt_make_context_loops(): async def crasher(): raise KeyError - with pytest.raises(_core.MultiError) as excinfo: + with pytest.raises(MultiError) as excinfo: async with _core.open_nursery() as nursery: nursery.start_soon(crasher) raise ValueError @@ -1615,7 +1619,7 @@ async def test_trivial_yields(): with _core.CancelScope() as cancel_scope: cancel_scope.cancel() - with pytest.raises(_core.MultiError) as excinfo: + with pytest.raises(MultiError) as excinfo: async with _core.open_nursery(): raise KeyError assert len(excinfo.value.exceptions) == 2 @@ -1705,7 +1709,7 @@ async def raise_keyerror_after_started(task_status=_core.TASK_STATUS_IGNORED): async with _core.open_nursery() as nursery: with _core.CancelScope() as cs: cs.cancel() - with pytest.raises(_core.MultiError) as excinfo: + with pytest.raises(MultiError) as excinfo: await nursery.start(raise_keyerror_after_started) assert {type(e) for e in excinfo.value.exceptions} == { _core.Cancelled, @@ -1820,7 +1824,7 @@ async def fail(): async with _core.open_nursery() as nursery: nursery.start_soon(fail) raise StopIteration - except _core.MultiError as e: + except MultiError as e: assert tuple(map(type, e.exceptions)) == (StopIteration, ValueError) @@ -1851,23 +1855,11 @@ def __aiter__(self): async def __anext__(self): nexts = self.nexts items = [None] * len(nexts) - got_stop = False - - def handle(exc): - nonlocal got_stop - if isinstance(exc, StopAsyncIteration): - got_stop = True - return None - else: # pragma: no cover - return exc - with _core.MultiError.catch(handle): - async with _core.open_nursery() as nursery: - for i, f in enumerate(nexts): - nursery.start_soon(self._accumulate, f, items, i) + async with _core.open_nursery() as nursery: + for i, f in enumerate(nexts): + nursery.start_soon(self._accumulate, f, items, i) - if got_stop: - raise StopAsyncIteration return items result = [] @@ -1889,7 +1881,7 @@ async def my_child_task(): async with _core.open_nursery() as nursery: nursery.start_soon(my_child_task) nursery.start_soon(my_child_task) - except _core.MultiError as exc: + except MultiError as exc: first_exc = exc.exceptions[0] assert isinstance(first_exc, KeyError) # The top frame in the exception traceback should be inside the child @@ -2231,7 +2223,7 @@ async def crasher(): with pytest.raises(ValueError): async with _core.open_nursery() as nursery: - # cover MultiError.filter and NurseryManager.__aexit__ + # cover NurseryManager.__aexit__ nursery.start_soon(crasher) gc.collect() @@ -2331,3 +2323,109 @@ async def task(): nursery.start_soon(task) nursery.cancel_scope.cancel() assert destroyed + + +def test_run_strict_exception_groups(): + """ + Test that nurseries respect the global context setting of strict_exception_groups. + """ + + async def main(): + async with _core.open_nursery(): + raise Exception("foo") + + with pytest.raises(MultiError) as exc: + _core.run(main, strict_exception_groups=True) + + assert len(exc.value.exceptions) == 1 + assert type(exc.value.exceptions[0]) is Exception + assert exc.value.exceptions[0].args == ("foo",) + + +def test_run_strict_exception_groups_nursery_override(): + """ + Test that a nursery can override the global context setting of + strict_exception_groups. + """ + + async def main(): + async with _core.open_nursery(strict_exception_groups=False): + raise Exception("foo") + + with pytest.raises(Exception, match="foo"): + _core.run(main, strict_exception_groups=True) + + +async def test_nursery_strict_exception_groups(): + """Test that strict exception groups can be enabled on a per-nursery basis.""" + with pytest.raises(MultiError) as exc: + async with _core.open_nursery(strict_exception_groups=True): + raise Exception("foo") + + assert len(exc.value.exceptions) == 1 + assert type(exc.value.exceptions[0]) is Exception + assert exc.value.exceptions[0].args == ("foo",) + + +async def test_nursery_collapse_strict(): + """ + Test that a single exception from a nested nursery with strict semantics doesn't get + collapsed when CancelledErrors are stripped from it. + """ + + async def raise_error(): + raise RuntimeError("test error") + + with pytest.raises(MultiError) as exc: + async with _core.open_nursery() as nursery: + nursery.start_soon(sleep_forever) + nursery.start_soon(raise_error) + async with _core.open_nursery(strict_exception_groups=True) as nursery2: + nursery2.start_soon(sleep_forever) + nursery2.start_soon(raise_error) + nursery.cancel_scope.cancel() + + exceptions = exc.value.exceptions + assert len(exceptions) == 2 + assert isinstance(exceptions[0], RuntimeError) + assert isinstance(exceptions[1], MultiError) + assert len(exceptions[1].exceptions) == 1 + assert isinstance(exceptions[1].exceptions[0], RuntimeError) + + +async def test_nursery_collapse_loose(): + """ + Test that a single exception from a nested nursery with loose semantics gets + collapsed when CancelledErrors are stripped from it. + """ + + async def raise_error(): + raise RuntimeError("test error") + + with pytest.raises(MultiError) as exc: + async with _core.open_nursery() as nursery: + nursery.start_soon(sleep_forever) + nursery.start_soon(raise_error) + async with _core.open_nursery() as nursery2: + nursery2.start_soon(sleep_forever) + nursery2.start_soon(raise_error) + nursery.cancel_scope.cancel() + + exceptions = exc.value.exceptions + assert len(exceptions) == 2 + assert isinstance(exceptions[0], RuntimeError) + assert isinstance(exceptions[1], RuntimeError) + + +async def test_cancel_scope_no_cancellederror(): + """ + Test that when a cancel scope encounters an exception group that does NOT contain + a Cancelled exception, it will NOT set the ``cancelled_caught`` flag. + """ + + with pytest.raises(ExceptionGroup): + with _core.CancelScope() as scope: + scope.cancel() + raise ExceptionGroup("test", [RuntimeError(), RuntimeError()]) + + assert not scope.cancelled_caught diff --git a/trio/_highlevel_open_tcp_listeners.py b/trio/_highlevel_open_tcp_listeners.py index 80f2c7a180..b650ac973f 100644 --- a/trio/_highlevel_open_tcp_listeners.py +++ b/trio/_highlevel_open_tcp_listeners.py @@ -5,6 +5,9 @@ import trio from . import socket as tsocket +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + # Default backlog size: # @@ -134,11 +137,13 @@ async def open_tcp_listeners(port, *, host=None, backlog=None): raise if unsupported_address_families and not listeners: - raise OSError( - errno.EAFNOSUPPORT, + msg = ( "This system doesn't support any of the kinds of " - "socket that that address could use", - ) from trio.MultiError(unsupported_address_families) + "socket that that address could use" + ) + raise OSError(errno.EAFNOSUPPORT, msg) from ExceptionGroup( + msg, unsupported_address_families + ) return listeners diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 545fac8641..740c0b1752 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -1,8 +1,14 @@ +import sys from contextlib import contextmanager import trio +from trio._core._multierror import MultiError from trio.socket import getaddrinfo, SOCK_STREAM, socket +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + + # Implementation of RFC 6555 "Happy eyeballs" # https://tools.ietf.org/html/rfc6555 # @@ -114,8 +120,10 @@ def close_all(): sock.close() except BaseException as exc: errs.append(exc) - if errs: - raise trio.MultiError(errs) + if len(errs) == 1: + raise errs[0] + elif errs: + raise MultiError(errs) def reorder_for_rfc_6555_section_5_4(targets): @@ -364,7 +372,7 @@ async def attempt_connect(socket_args, sockaddr, attempt_failed): msg = "all attempts to connect to {} failed".format( format_host_port(host, port) ) - raise OSError(msg) from trio.MultiError(oserrors) + raise OSError(msg) from ExceptionGroup(msg, oserrors) else: stream = trio.SocketStream(winning_socket) open_sockets.remove(winning_socket) diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index d5fc576ec5..103984739e 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -1,3 +1,5 @@ +import sys + import pytest import socket as stdlib_socket @@ -11,6 +13,9 @@ from .. import socket as tsocket from .._core.tests.tutil import slow, creates_ipv6, binds_ipv6 +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + async def test_open_tcp_listeners_basic(): listeners = await open_tcp_listeners(0) @@ -239,7 +244,7 @@ async def test_open_tcp_listeners_some_address_families_unavailable( await open_tcp_listeners(80, host="example.org") assert "This system doesn't support" in str(exc_info.value) - if isinstance(exc_info.value.__cause__, trio.MultiError): + if isinstance(exc_info.value.__cause__, BaseExceptionGroup): for subexc in exc_info.value.__cause__.exceptions: assert "nope" in str(subexc) else: diff --git a/trio/tests/test_highlevel_open_tcp_stream.py b/trio/tests/test_highlevel_open_tcp_stream.py index eaaff3e17f..0f3b6a0baf 100644 --- a/trio/tests/test_highlevel_open_tcp_stream.py +++ b/trio/tests/test_highlevel_open_tcp_stream.py @@ -13,6 +13,9 @@ format_host_port, ) +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + def test_close_all(): class CloseMe: @@ -436,7 +439,7 @@ async def test_all_fail(autojump_clock): expect_error=OSError, ) assert isinstance(exc, OSError) - assert isinstance(exc.__cause__, trio.MultiError) + assert isinstance(exc.__cause__, BaseExceptionGroup) assert len(exc.__cause__.exceptions) == 4 assert trio.current_time() == (0.1 + 0.2 + 10) assert scenario.connect_times == { @@ -556,7 +559,7 @@ async def test_cancel(autojump_clock): ("3.3.3.3", 10, "success"), ("4.4.4.4", 10, "success"), ], - expect_error=trio.MultiError, + expect_error=BaseExceptionGroup, ) # What comes out should be 1 or more Cancelled errors that all belong # to this cancel_scope; this is the easiest way to check that