From 2b5800f518c582d391cd22d5b63edb1bd2e7f64a Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 15 Apr 2021 13:19:22 +0100 Subject: [PATCH 01/40] bpo-45292: [PEP 654] added ExceptionGroup and BaseExceptionGroup (did not yet update traceback formatting) --- Doc/data/stable_abi.dat | 2 + Include/cpython/pyerrors.h | 6 + Include/pyerrors.h | 2 + Lib/test/exception_hierarchy.txt | 2 + Lib/test/test_descr.py | 6 +- Lib/test/test_doctest.py | 2 +- Lib/test/test_exception_group.py | 739 +++++++++++++++++++++++++++++++ Lib/test/test_pickle.py | 4 +- Misc/stable_abi.txt | 4 + Objects/exceptions.c | 455 +++++++++++++++++++ PC/python3dll.c | 2 + 11 files changed, 1221 insertions(+), 3 deletions(-) create mode 100644 Lib/test/test_exception_group.py diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 46ee321b660c3a..ff89a1fff8daf5 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -189,6 +189,7 @@ var,PyExc_ArithmeticError,3.2, var,PyExc_AssertionError,3.2, var,PyExc_AttributeError,3.2, var,PyExc_BaseException,3.2, +var,PyExc_BaseExceptionGroup,3.11, var,PyExc_BlockingIOError,3.7, var,PyExc_BrokenPipeError,3.7, var,PyExc_BufferError,3.2, @@ -203,6 +204,7 @@ var,PyExc_EOFError,3.2, var,PyExc_EncodingWarning,3.10, var,PyExc_EnvironmentError,3.2, var,PyExc_Exception,3.2, +var,PyExc_ExceptionGroup,3.11, var,PyExc_FileExistsError,3.7, var,PyExc_FileNotFoundError,3.7, var,PyExc_FloatingPointError,3.2, diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index a3ec5afdb7c788..28ab565dde4237 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -14,6 +14,12 @@ typedef struct { PyException_HEAD } PyBaseExceptionObject; +typedef struct { + PyException_HEAD + PyObject *msg; + PyObject *excs; +} PyBaseExceptionGroupObject; + typedef struct { PyException_HEAD PyObject *msg; diff --git a/Include/pyerrors.h b/Include/pyerrors.h index c6c443a2d7d0f0..b2693c969d7176 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -65,6 +65,8 @@ PyAPI_FUNC(const char *) PyExceptionClass_Name(PyObject *); PyAPI_DATA(PyObject *) PyExc_BaseException; PyAPI_DATA(PyObject *) PyExc_Exception; +PyAPI_DATA(PyObject *) PyExc_BaseExceptionGroup; +PyAPI_DATA(PyObject *) PyExc_ExceptionGroup; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 PyAPI_DATA(PyObject *) PyExc_StopAsyncIteration; #endif diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index cf54454e71afac..5c0bfda373794c 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -2,7 +2,9 @@ BaseException ├── SystemExit ├── KeyboardInterrupt ├── GeneratorExit + ├── BaseExceptionGroup └── Exception + ├── ExceptionGroup [BaseExceptionGroup] ├── StopIteration ├── StopAsyncIteration ├── ArithmeticError diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index a5404b30d2459a..a4131bec602ea3 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4032,7 +4032,11 @@ def test_builtin_bases(self): for tp in builtin_types: object.__getattribute__(tp, "__bases__") if tp is not object: - self.assertEqual(len(tp.__bases__), 1, tp) + if tp is ExceptionGroup: + num_bases = 2 + else: + num_bases = 1 + self.assertEqual(len(tp.__bases__), num_bases, tp) class L(list): pass diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 3524a0a797c41a..603eb432567509 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -668,7 +668,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 820 < len(tests) < 840 # approximate number of objects with docstrings + >>> 820 < len(tests) < 845 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py new file mode 100644 index 00000000000000..0c11c52e642734 --- /dev/null +++ b/Lib/test/test_exception_group.py @@ -0,0 +1,739 @@ +import collections.abc +import traceback +import types +import unittest + + +class TestExceptionGroupTypeHierarchy(unittest.TestCase): + def test_exception_group_types(self): + self.assertTrue(issubclass(ExceptionGroup, Exception)) + self.assertTrue(issubclass(ExceptionGroup, BaseExceptionGroup)) + self.assertTrue(issubclass(BaseExceptionGroup, BaseException)) + + +class BadConstructorArgs(unittest.TestCase): + def test_bad_EG_construction__too_many_args(self): + MSG = 'function takes exactly 2 arguments' + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup('no errors') + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup([ValueError('no msg')]) + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup('eg', [ValueError('too')], [TypeError('many')]) + + def test_bad_EG_construction__bad_message(self): + MSG = 'Expected a message' + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup(ValueError(12), SyntaxError('bad syntax')) + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup(None, [ValueError(12)]) + + def test_bad_EG_construction__bad_excs_sequence(self): + MSG = 'Expected a non-empty sequence' + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup('errors not sequence', {ValueError(42)}) + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup("eg", []) + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup("eg", None) + + def test_bad_EG_construction__nested_non_exceptions(self): + MSG = 'Nested exceptions must derive from BaseException' + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup('bad error', ["not an exception"]) + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup('eg1', + [ExceptionGroup('eg2', ["not an exception"])]) + + +class InstanceCreation(unittest.TestCase): + def test_EG_wraps_Exceptions__creates_EG(self): + excs = [ValueError(1), TypeError(2)] + self.assertIs(type(ExceptionGroup("eg", excs)), ExceptionGroup) + + def test_BEG_wraps_Exceptions__creates_EG(self): + excs = [ValueError(1), TypeError(2)] + self.assertIs(type(BaseExceptionGroup("beg", excs)), ExceptionGroup) + + def test_EG_wraps_BaseException__raises_TypeError(self): + MSG= "Cannot nest BaseExceptions in an ExceptionGroup" + with self.assertRaisesRegex(TypeError, MSG): + eg = ExceptionGroup("eg", [ValueError(1), KeyboardInterrupt(2)]) + + def test_BEG_wraps_BaseException__creates_BEG(self): + beg = BaseExceptionGroup("beg", [ValueError(1), KeyboardInterrupt(2)]) + self.assertIs(type(beg), BaseExceptionGroup) + + def test_EG_subclass_wraps_anything(self): + class MyEG(ExceptionGroup): + pass + self.assertIs( + type(MyEG("eg", [ValueError(12), TypeError(42)])), MyEG) + self.assertIs( + type(MyEG("eg", [ValueError(12), KeyboardInterrupt(42)])), MyEG) + + def test_BEG_subclass_wraps_anything(self): + class MyBEG(BaseExceptionGroup): + pass + self.assertIs( + type(MyBEG("eg", [ValueError(12), TypeError(42)])), MyBEG) + self.assertIs( + type(MyBEG("eg", [ValueError(12), KeyboardInterrupt(42)])), MyBEG) + + +def leaf_generator(exc, tbs=None): + if tbs is None: + tbs = [] + tbs.append(exc.__traceback__) + if isinstance(exc, BaseExceptionGroup): + for e in exc.exceptions: + yield from leaf_generator(e, tbs) + else: + # exc is a leaf exception and its traceback + # is the concatenation of the traceback + # segments in tbs + yield exc, tbs + tbs.pop() + + +def tb_funcnames(tbs): + def names_list(tb): + if tb is None: + return None + else: + return [f.name for f in traceback.extract_tb(tb)] + + if isinstance(tbs, types.TracebackType): + return names_list(tbs) + else: + return [names_list(tb) for tb in tbs] + + +def tbs_for_leaf(leaf, eg): + for e, tbs in leaf_generator(eg): + if e == leaf: + return tbs + + +class ExceptionGroupTestBase(unittest.TestCase): + def assertMatchesTemplate(self, exc, exc_type, template): + """ Assert that the exception matches the template """ + if exc_type is not None: + self.assertIs(type(exc), exc_type) + + if isinstance(exc, BaseExceptionGroup): + self.assertIsInstance(template, collections.abc.Sequence) + self.assertEqual(len(exc.exceptions), len(template)) + for e, t in zip(exc.exceptions, template): + self.assertMatchesTemplate(e, None, t) + else: + self.assertIsInstance(template, BaseException) + self.assertEqual(type(exc), type(template)) + self.assertEqual(exc.args, template.args) + + +class ExceptionGroupBasicsTests(ExceptionGroupTestBase): + def setUp(self): + # simple ExceptionGroup (without nesting) + + def do_raise(type, arg, cause=None): + if cause is None: + raise type(arg) + else: + raise type(arg) from cause + + def getTypeError(v): + try: + try: + do_raise(OSError, "context for TypeError") + except: + do_raise(TypeError, v) + except Exception as e: + return e + + def getValueError(v): + try: + try: + do_raise(MemoryError, "context and cause for ValueError") + except MemoryError as e: + do_raise(ValueError, v, cause=e) + except Exception as e: + return e + + def simple_group(): + try: + raise ExceptionGroup( + 'simple EG', + [getValueError(1), getTypeError(int), getValueError(2)]) + except ExceptionGroup as e: + return e + + try: + raise simple_group() + except Exception as e: + self.eg = e + + self.eg_template = [ValueError(1), TypeError(int), ValueError(2)] + + self.root_tb_fnames = ['setUp', 'simple_group'] + self.value_error_tb_fnames = ['getValueError', 'do_raise'] + self.type_error_tb_fnames = ['getTypeError', 'do_raise'] + + def test_basics_ExceptionGroup_fields(self): + eg = self.eg + + self.assertMatchesTemplate(eg, ExceptionGroup, self.eg_template) + + # check msg + self.assertEqual(eg.message, 'simple EG') + self.assertEqual(eg.args[0], 'simple EG') + + # check cause and context + self.assertIsInstance(eg.exceptions[0], ValueError) + self.assertIsInstance(eg.exceptions[0].__cause__, MemoryError) + self.assertIsInstance(eg.exceptions[0].__context__, MemoryError) + self.assertIsInstance(eg.exceptions[1], TypeError) + self.assertIsNone(eg.exceptions[1].__cause__) + self.assertIsInstance(eg.exceptions[1].__context__, OSError) + self.assertIsInstance(eg.exceptions[2], ValueError) + self.assertIsInstance(eg.exceptions[2].__cause__, MemoryError) + self.assertIsInstance(eg.exceptions[2].__context__, MemoryError) + + # check tracebacks + self.assertSequenceEqual( + tb_funcnames(eg.__traceback__), self.root_tb_fnames) + + self.assertSequenceEqual( + tb_funcnames(eg.exceptions[0].__traceback__), + self.value_error_tb_fnames) + self.assertSequenceEqual( + tb_funcnames(eg.exceptions[1].__traceback__), + self.type_error_tb_fnames) + self.assertSequenceEqual( + tb_funcnames(eg.exceptions[2].__traceback__), + self.value_error_tb_fnames) + + + def test_basics_ExceptionGroup_fields_are_read_only(self): + eg = self.eg + self.assertEqual(type(eg.exceptions), tuple) + + _ = eg.message + with self.assertRaises(AttributeError): + eg.message = "new msg" + _ = eg.exceptions + with self.assertRaises(AttributeError): + eg.exceptions = [OSError('xyz')] + + def test_basics_leaf_generator(self): + eg = self.eg + self.assertSequenceEqual( + [e for e, _ in leaf_generator(eg)], eg.exceptions) + + self.assertSequenceEqual( + [tb_funcnames(tbs) for _, tbs in leaf_generator(eg)], + [[self.root_tb_fnames, self.value_error_tb_fnames], + [self.root_tb_fnames, self.type_error_tb_fnames], + [self.root_tb_fnames, self.value_error_tb_fnames]]) + + def test_basics_tbs_for_leaf(self): + eg = self.eg + for e, tbs in leaf_generator(eg): + self.assertSequenceEqual(tbs, tbs_for_leaf(e, eg)) + + def test_basics_subgroup_split__bad_arg_type(self): + bad_args = ["bad arg", + OSError('instance not type'), + [OSError('instance not type')],] + for arg in bad_args: + with self.assertRaises(TypeError): + self.eg.subgroup(arg) + with self.assertRaises(TypeError): + self.eg.split(arg) + + def test_basics_subgroup_by_type__passthrough(self): + eg = self.eg + self.assertIs(eg, eg.subgroup(BaseException)) + self.assertIs(eg, eg.subgroup(Exception)) + self.assertIs(eg, eg.subgroup(BaseExceptionGroup)) + self.assertIs(eg, eg.subgroup(ExceptionGroup)) + + def test_basics_subgroup_by_type__no_match(self): + self.assertIsNone(self.eg.subgroup(OSError)) + + def test_basics_subgroup_by_type__match(self): + eg = self.eg + testcases = [ + # (match_type, result_template) + (ValueError, [ValueError(1), ValueError(2)]), + (TypeError, [TypeError(int)]), + ((ValueError, TypeError), self.eg_template)] + + for match_type, template in testcases: + subeg = eg.subgroup(match_type) + self.assertEqual(subeg.message, eg.message) + self.assertMatchesTemplate(subeg, ExceptionGroup, template) + + def test_basics_subgroup_by_predicate__passthrough(self): + self.assertIs(self.eg, self.eg.subgroup(lambda e: True)) + + def test_basics_subgroup_by_predicate__no_match(self): + self.assertIsNone(self.eg.subgroup(lambda e: False)) + + def test_basics_subgroup_by_predicate__match(self): + eg = self.eg + testcases = [ + # (match_type, result_template) + (ValueError, [ValueError(1), ValueError(2)]), + (TypeError, [TypeError(int)]), + ((ValueError, TypeError), self.eg_template)] + + for match_type, template in testcases: + subeg = eg.subgroup(lambda e: isinstance(e, match_type)) + self.assertEqual(subeg.message, eg.message) + self.assertMatchesTemplate(subeg, ExceptionGroup, template) + + def test_basics_split_by_type__passthrough(self): + for E in [BaseException, Exception, + BaseExceptionGroup, ExceptionGroup]: + match, rest = self.eg.split(E) + self.assertMatchesTemplate(match, ExceptionGroup, self.eg_template) + self.assertIsNone(rest) + + def test_basics_split_by_type__no_match(self): + match, rest = self.eg.split(OSError) + self.assertIsNone(match) + self.assertMatchesTemplate(rest, ExceptionGroup, self.eg_template) + + def test_basics_split_by_type__match(self): + eg = self.eg + testcases = [ + # (matcher, match_template, rest_template) + (ValueError, [ValueError(1), ValueError(2)], [TypeError(int)]), + (TypeError, [TypeError(int)], [ValueError(1), ValueError(2)]), + ((ValueError, TypeError), self.eg_template, None)] + for match_type, match_template, rest_template in testcases: + match, rest = eg.split(match_type) + self.assertEqual(match.message, eg.message) + self.assertMatchesTemplate( + match, ExceptionGroup, match_template) + if rest_template is not None: + self.assertEqual(rest.message, eg.message) + self.assertMatchesTemplate( + rest, ExceptionGroup, rest_template) + + def test_basics_split_by_predicate__passthrough(self): + match, rest = self.eg.split(lambda e: True) + self.assertMatchesTemplate(match, ExceptionGroup, self.eg_template) + self.assertIsNone(rest) + + def test_basics_split_by_predicate__no_match(self): + match, rest = self.eg.split(lambda e: False) + self.assertIsNone(match) + self.assertMatchesTemplate(rest, ExceptionGroup, self.eg_template) + + def test_basics_split_by_predicate__match(self): + eg = self.eg + testcases = [ + # (matcher, match_template, rest_template) + (ValueError, [ValueError(1), ValueError(2)], [TypeError(int)]), + (TypeError, [TypeError(int)], [ValueError(1), ValueError(2)]), + ((ValueError, TypeError), self.eg_template, None)] + for match_type, match_template, rest_template in testcases: + match, rest = eg.split(lambda e: isinstance(e, match_type)) + self.assertEqual(match.message, eg.message) + self.assertMatchesTemplate( + match, ExceptionGroup, match_template) + if rest_template is not None: + self.assertEqual(rest.message, eg.message) + self.assertMatchesTemplate( + rest, ExceptionGroup, rest_template) + + +class NestedExceptionGroupBasicsTest(ExceptionGroupTestBase): + def setUp(self): + def raiseVE(v): + raise ValueError(v) + + def raiseTE(t): + raise TypeError(t) + + def nested_group(): + try: + try: + raiseTE(bytes) + except TypeError as e: + raise ExceptionGroup("nested", [e]) + except ExceptionGroup as e: + excs = [e] + try: + raiseVE(1) + except ValueError as e: + excs.append(e) + raise ExceptionGroup("root", excs) + + try: + nested_group() + except ExceptionGroup as eg: + self.eg = eg + + def test_nested_group_matches_template(self): + self.assertMatchesTemplate( + self.eg, ExceptionGroup, [[TypeError(bytes)], ValueError(1)]) + + def test_nested_group_tracebacks(self): + eg = self.eg + def check(exc, expected): + self.assertEqual(tb_funcnames(exc.__traceback__), expected) + + check(eg, ['setUp', 'nested_group']) + check(eg.exceptions[0], ['nested_group']) + check(eg.exceptions[1], ['nested_group', 'raiseVE']) + check(eg.exceptions[0].exceptions[0], ['nested_group', 'raiseTE']) + + def test_iteration_full_tracebacks(self): + eg = self.eg + # check that iteration over leaves + # produces the expected tracebacks + self.assertEqual(len(list(leaf_generator(eg))), 2) + + expected_tbs = [ + [['setUp', 'nested_group'], + ['nested_group'], + ['nested_group', 'raiseTE']], + [['setUp', 'nested_group'], + ['nested_group', 'raiseVE']]] + + for (i, (e, tbs)) in enumerate(leaf_generator(eg)): + self.assertSequenceEqual( + expected_tbs[i], [tb_funcnames(tb) for tb in tbs]) + + +class ExceptionGroupSplitTests(ExceptionGroupTestBase): + def _split_exception_group(self, eg, types): + """ Split an EG and do some sanity checks on the result """ + self.assertIsInstance(eg, BaseExceptionGroup) + + match, rest = eg.split(types) + sg = eg.subgroup(types) + + if match is not None: + self.assertIsInstance(match, BaseExceptionGroup) + for e,_ in leaf_generator(match): + self.assertIsInstance(e, types) + + self.assertIsNotNone(sg) + self.assertIsInstance(sg, BaseExceptionGroup) + for e,_ in leaf_generator(sg): + self.assertIsInstance(e, types) + + if rest is not None: + self.assertIsInstance(rest, BaseExceptionGroup) + + def leaves(exc): + return [] if exc is None else [e for e,_ in leaf_generator(exc)] + + # match and subgroup have the same leaves + self.assertSequenceEqual(leaves(match), leaves(sg)) + + match_leaves = leaves(match) + rest_leaves = leaves(rest) + # each leaf exception of eg and exactly one of match and rest + self.assertEqual(len(leaves(eg)), len(leaves(match)) + len(leaves(rest))) + for e in leaves(eg): + self.assertNotEqual( + match and e in match_leaves, + rest and e in rest_leaves) + + # message, cause and context equal to eg + for part in [match, rest, sg]: + if part is not None: + self.assertEqual(eg.message, part.message) + self.assertIs(eg.__cause__, part.__cause__) + self.assertIs(eg.__context__, part.__context__) + self.assertIs(eg.__traceback__, part.__traceback__) + + # full tracebacks match + for part in [match, rest, sg]: + for e in leaves(part): + self.assertSequenceEqual( + tb_funcnames(tbs_for_leaf(e, eg)), + tb_funcnames(tbs_for_leaf(e, part))) + + return match, rest + + def test_split_by_type(self): + class MyExceptionGroup(ExceptionGroup): + pass + + def raiseVE(v): + raise ValueError(v) + + def raiseTE(t): + raise TypeError(t) + + def nested_group(): + def level1(i): + excs = [] + for f, arg in [(raiseVE, i), (raiseTE, int), (raiseVE, i+1)]: + try: + f(arg) + except Exception as e: + excs.append(e) + raise ExceptionGroup('msg1', excs) + + def level2(i): + excs = [] + for f, arg in [(level1, i), (level1, i+1), (raiseVE, i+2)]: + try: + f(arg) + except Exception as e: + excs.append(e) + raise MyExceptionGroup('msg2', excs) + + def level3(i): + excs = [] + for f, arg in [(level2, i+1), (raiseVE, i+2)]: + try: + f(arg) + except Exception as e: + excs.append(e) + raise ExceptionGroup('msg3', excs) + + level3(5) + + try: + nested_group() + except ExceptionGroup as e: + eg = e + + eg_template = [ + [ + [ValueError(6), TypeError(int), ValueError(7)], + [ValueError(7), TypeError(int), ValueError(8)], + ValueError(8), + ], + ValueError(7)] + + valueErrors_template = [ + [ + [ValueError(6), ValueError(7)], + [ValueError(7), ValueError(8)], + ValueError(8), + ], + ValueError(7)] + + typeErrors_template = [[[TypeError(int)], [TypeError(int)]]] + + self.assertMatchesTemplate(eg, ExceptionGroup, eg_template) + + # Match Nothing + match, rest = self._split_exception_group(eg, SyntaxError) + self.assertIsNone(match) + self.assertMatchesTemplate(rest, ExceptionGroup, eg_template) + + # Match Everything + match, rest = self._split_exception_group(eg, BaseException) + self.assertMatchesTemplate(match, ExceptionGroup, eg_template) + self.assertIsNone(rest) + match, rest = self._split_exception_group(eg, (ValueError, TypeError)) + self.assertMatchesTemplate(match, ExceptionGroup, eg_template) + self.assertIsNone(rest) + + # Match ValueErrors + match, rest = self._split_exception_group(eg, ValueError) + self.assertMatchesTemplate(match, ExceptionGroup, valueErrors_template) + self.assertMatchesTemplate(rest, ExceptionGroup, typeErrors_template) + + # Match TypeErrors + match, rest = self._split_exception_group(eg, (TypeError, SyntaxError)) + self.assertMatchesTemplate(match, ExceptionGroup, typeErrors_template) + self.assertMatchesTemplate(rest, ExceptionGroup, valueErrors_template) + + # Match ExceptionGroup + match, rest = eg.split(ExceptionGroup) + self.assertIs(match, eg) + self.assertIsNone(rest) + + # Match MyExceptionGroup (ExceptionGroup subclass) + match, rest = eg.split(MyExceptionGroup) + self.assertMatchesTemplate(match, ExceptionGroup, [eg_template[0]]) + self.assertMatchesTemplate(rest, ExceptionGroup, [eg_template[1]]) + + def test_split_BaseExceptionGroup(self): + def exc(ex): + try: + raise ex + except BaseException as e: + return e + + try: + raise BaseExceptionGroup( + "beg", [exc(ValueError(1)), exc(KeyboardInterrupt(2))]) + except BaseExceptionGroup as e: + beg = e + + # Match Nothing + match, rest = self._split_exception_group(beg, TypeError) + self.assertIsNone(match) + self.assertMatchesTemplate( + rest, BaseExceptionGroup, [ValueError(1), KeyboardInterrupt(2)]) + + # Match Everything + match, rest = self._split_exception_group(beg, (ValueError, KeyboardInterrupt)) + self.assertMatchesTemplate( + match, BaseExceptionGroup, [ValueError(1), KeyboardInterrupt(2)]) + self.assertIsNone(rest) + + # Match ValueErrors + match, rest = self._split_exception_group(beg, ValueError) + self.assertMatchesTemplate(match, ExceptionGroup, [ValueError(1)]) + self.assertMatchesTemplate(rest, BaseExceptionGroup, [KeyboardInterrupt(2)]) + + # Match KeyboardInterrupts + match, rest = self._split_exception_group(beg, KeyboardInterrupt) + self.assertMatchesTemplate(match, BaseExceptionGroup, [KeyboardInterrupt(2)]) + self.assertMatchesTemplate(rest, ExceptionGroup, [ValueError(1)]) + + def test_split_ExceptionGroup_subclass_no_derive_no_new_override(self): + class EG(ExceptionGroup): + pass + + try: + try: + try: + raise TypeError(2) + except TypeError as te: + raise EG("nested", [te]) + except EG as nested: + try: + raise ValueError(1) + except ValueError as ve: + raise EG("eg", [ve, nested]) + except EG as e: + eg = e + + self.assertMatchesTemplate(eg, EG, [ValueError(1), [TypeError(2)]]) + + # Match Nothing + match, rest = self._split_exception_group(eg, OSError) + self.assertIsNone(match) + self.assertMatchesTemplate( + rest, ExceptionGroup, [ValueError(1), [TypeError(2)]]) + + # Match Everything + match, rest = self._split_exception_group(eg, (ValueError, TypeError)) + self.assertMatchesTemplate( + match, ExceptionGroup, [ValueError(1), [TypeError(2)]]) + self.assertIsNone(rest) + + # Match ValueErrors + match, rest = self._split_exception_group(eg, ValueError) + self.assertMatchesTemplate(match, ExceptionGroup, [ValueError(1)]) + self.assertMatchesTemplate(rest, ExceptionGroup, [[TypeError(2)]]) + + # Match TypeErrors + match, rest = self._split_exception_group(eg, TypeError) + self.assertMatchesTemplate(match, ExceptionGroup, [[TypeError(2)]]) + self.assertMatchesTemplate(rest, ExceptionGroup, [ValueError(1)]) + + def test_split_BaseExceptionGroup_subclass_no_derive_new_override(self): + class EG(BaseExceptionGroup): + def __new__(cls, message, excs, unused): + # The "unused" arg is here to show that split() doesn't call + # the actual class constructor from the default derive() + # implementation (it would fail on unused arg if so because + # it assumes the BaseExceptionGroup.__new__ signature). + return super().__new__(cls, message, excs) + + try: + raise EG("eg", [ValueError(1), KeyboardInterrupt(2)], "unused") + except EG as e: + eg = e + + self.assertMatchesTemplate( + eg, EG, [ValueError(1), KeyboardInterrupt(2)]) + + # Match Nothing + match, rest = self._split_exception_group(eg, OSError) + self.assertIsNone(match) + self.assertMatchesTemplate( + rest, BaseExceptionGroup, [ValueError(1), KeyboardInterrupt(2)]) + + # Match Everything + match, rest = self._split_exception_group(eg, (ValueError, KeyboardInterrupt)) + self.assertMatchesTemplate( + match, BaseExceptionGroup, [ValueError(1), KeyboardInterrupt(2)]) + self.assertIsNone(rest) + + # Match ValueErrors + match, rest = self._split_exception_group(eg, ValueError) + self.assertMatchesTemplate(match, ExceptionGroup, [ValueError(1)]) + self.assertMatchesTemplate( + rest, BaseExceptionGroup, [KeyboardInterrupt(2)]) + + # Match KeyboardInterrupt + match, rest = self._split_exception_group(eg, KeyboardInterrupt) + self.assertMatchesTemplate( + match, BaseExceptionGroup, [KeyboardInterrupt(2)]) + self.assertMatchesTemplate(rest, ExceptionGroup, [ValueError(1)]) + + def test_split_ExceptionGroup_subclass_derive_and_new_overrides(self): + class EG(ExceptionGroup): + def __new__(cls, message, excs, code): + obj = super().__new__(cls, message, excs) + obj.code = code + return obj + + def derive(self, excs): + return EG(self.message, excs, self.code) + + try: + try: + try: + raise TypeError(2) + except TypeError as te: + raise EG("nested", [te], 101) + except EG as nested: + try: + raise ValueError(1) + except ValueError as ve: + raise EG("eg", [ve, nested], 42) + except EG as e: + eg = e + + self.assertMatchesTemplate(eg, EG, [ValueError(1), [TypeError(2)]]) + + # Match Nothing + match, rest = self._split_exception_group(eg, OSError) + self.assertIsNone(match) + self.assertMatchesTemplate(rest, EG, [ValueError(1), [TypeError(2)]]) + self.assertEqual(rest.code, 42) + self.assertEqual(rest.exceptions[1].code, 101) + + # Match Everything + match, rest = self._split_exception_group(eg, (ValueError, TypeError)) + self.assertMatchesTemplate(match, EG, [ValueError(1), [TypeError(2)]]) + self.assertEqual(match.code, 42) + self.assertEqual(match.exceptions[1].code, 101) + self.assertIsNone(rest) + + # Match ValueErrors + match, rest = self._split_exception_group(eg, ValueError) + self.assertMatchesTemplate(match, EG, [ValueError(1)]) + self.assertEqual(match.code, 42) + self.assertMatchesTemplate(rest, EG, [[TypeError(2)]]) + self.assertEqual(rest.code, 42) + self.assertEqual(rest.exceptions[0].code, 101) + + # Match TypeErrors + match, rest = self._split_exception_group(eg, TypeError) + self.assertMatchesTemplate(match, EG, [[TypeError(2)]]) + self.assertEqual(match.code, 42) + self.assertEqual(match.exceptions[0].code, 101) + self.assertMatchesTemplate(rest, EG, [ValueError(1)]) + self.assertEqual(rest.code, 42) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 8775ff4b79157d..412949134986ef 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -483,6 +483,8 @@ def test_exceptions(self): self.assertEqual(reverse_mapping('builtins', 'OSError'), ('exceptions', 'OSError')) + exceptions_added_in_python_3 = [ + 'BaseExceptionGroup', 'ExceptionGroup'] for name, exc in get_exceptions(builtins): with self.subTest(name): if exc in (BlockingIOError, @@ -499,7 +501,7 @@ def test_exceptions(self): ('exceptions', 'ImportError')) self.assertEqual(mapping('exceptions', name), ('exceptions', name)) - else: + elif not name in exceptions_added_in_python_3: self.assertEqual(reverse_mapping('builtins', name), ('exceptions', name)) self.assertEqual(mapping('exceptions', name), diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt index 23e5b96a0e8a78..1383707cf73619 100644 --- a/Misc/stable_abi.txt +++ b/Misc/stable_abi.txt @@ -619,6 +619,8 @@ data PyExc_AttributeError added 3.2 data PyExc_BaseException added 3.2 +data PyExc_BaseExceptionGroup + added 3.11 data PyExc_BufferError added 3.2 data PyExc_BytesWarning @@ -631,6 +633,8 @@ data PyExc_EnvironmentError added 3.2 data PyExc_Exception added 3.2 +data PyExc_ExceptionGroup + added 3.11 data PyExc_FloatingPointError added 3.2 data PyExc_FutureWarning diff --git a/Objects/exceptions.c b/Objects/exceptions.c index a9ea42c98422d9..0f826a4c5e01ea 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -629,6 +629,454 @@ ComplexExtendsException(PyExc_BaseException, SystemExit, SystemExit, 0, 0, SystemExit_members, 0, 0, "Request to exit from the interpreter."); +/* + * BaseExceptionGroup extends BaseException + * ExceptionGroup extends BaseExceptionGroup and Exception + */ + +PyObject *PyExc_ExceptionGroup; + +static inline PyBaseExceptionGroupObject* +_PyBaseExceptionGroupObject_cast(PyObject *exc) +{ + assert(PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_BaseExceptionGroup)); + return (PyBaseExceptionGroupObject *)exc; +} + +static PyObject * +BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *msg = NULL; + PyObject *excs = NULL; + + if (!PyArg_ParseTuple(args, "OO", &msg, &excs)) { + return NULL; + } + if (!PyUnicode_CheckExact(msg)) { + PyErr_SetString(PyExc_TypeError, "Expected a message"); + return NULL; + } + Py_ssize_t numexcs = -1; + if (PySequence_Check(excs)) { + numexcs = PySequence_Length(excs); + } + if (numexcs <= 0) { + PyErr_SetString(PyExc_TypeError, "Expected a non-empty sequence"); + return NULL; + } + + int nested_base_exceptions = 0; + for (Py_ssize_t i = 0; i < numexcs; i++) { + PyObject *exc = PySequence_GetItem(excs, i); + if (!exc) { + return NULL; + } + if (!PyExceptionInstance_Check(exc)) { + PyErr_SetString(PyExc_TypeError, + "Nested exceptions must derive from BaseException"); + Py_DECREF(exc); + return NULL; + } + int is_exception = PyObject_IsInstance(exc, PyExc_Exception); + Py_DECREF(exc); + if (is_exception < 0) { + return NULL; + } + else if (is_exception == 0) { + nested_base_exceptions = 1; + } + } + + PyTypeObject *cls = type; + if (cls == (PyTypeObject*)PyExc_ExceptionGroup) { + if (nested_base_exceptions) { + PyErr_SetString(PyExc_TypeError, + "Cannot nest BaseExceptions in an ExceptionGroup"); + return NULL; + } + } + else if (cls == (PyTypeObject*)PyExc_BaseExceptionGroup) { + if (!nested_base_exceptions) { + /* All nested exceptions are Exception subclasses, + * wrap them in an ExceptionGroup + */ + cls = (PyTypeObject*)PyExc_ExceptionGroup; + } + } + else { + /* Do nothing - we don't interfere with subclasses */ + } + + PyBaseExceptionGroupObject *self = + _PyBaseExceptionGroupObject_cast(BaseException_new(cls, args, kwds)); + self->msg = Py_NewRef(msg); + self->excs = PySequence_Tuple(excs); + return (PyObject*)self; +} + +static int +BaseExceptionGroup_init(PyBaseExceptionGroupObject *self, + PyObject *args, PyObject *kwds) +{ + if (!_PyArg_NoKeywords(Py_TYPE(self)->tp_name, kwds)) { + return -1; + } + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) { + return -1; + } + return 0; +} + +static int +BaseExceptionGroup_clear(PyBaseExceptionGroupObject *self) +{ + Py_CLEAR(self->msg); + Py_CLEAR(self->excs); + return BaseException_clear((PyBaseExceptionObject *)self); +} + +static void +BaseExceptionGroup_dealloc(PyBaseExceptionGroupObject *self) +{ + _PyObject_GC_UNTRACK(self); + BaseExceptionGroup_clear(self); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int +BaseExceptionGroup_traverse(PyBaseExceptionGroupObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->msg); + Py_VISIT(self->excs); + return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg); +} + +static PyObject * +BaseExceptionGroup_str(PyBaseExceptionGroupObject *self) +{ + if (self->msg && PyUnicode_CheckExact(self->msg)) { + Py_INCREF(self->msg); + return self->msg; + } + else { + return BaseException_str((PyBaseExceptionObject *)self); + } +} + +static PyObject * +BaseExceptionGroup_derive(PyObject *self_, PyObject *args) { + PyBaseExceptionGroupObject *self = _PyBaseExceptionGroupObject_cast(self_); + PyObject *excs = NULL; + if (!PyArg_ParseTuple(args, "O", &excs)) { + return NULL; + } + PyObject *init_args = PyTuple_Pack(2, self->msg, excs); + if (!init_args) { + return NULL; + } + PyObject *eg = PyObject_CallObject( + PyExc_BaseExceptionGroup, init_args); + Py_DECREF(init_args); + return eg; +} + +static PyObject* +exceptiongroup_subset(PyBaseExceptionGroupObject *orig, PyObject *excs) +{ + /* Return an ExceptionGroup wrapping excs with metadata from orig. + + This function is used by split() to construct the match/rest parts, + so excs is the matching or non-matching sub-sequence of orig->excs + (this function does not verify that it is a subsequence). + */ + Py_ssize_t num_excs = PySequence_Size(excs); + if (num_excs < 0) { + return NULL; + } + else if (num_excs == 0) { + return Py_NewRef(Py_None); + } + + PyObject *eg = PyObject_CallMethod( + (PyObject*)orig, "derive", "(O)", excs); + if (!eg) { + return NULL; + } + + if (!PyObject_TypeCheck(eg, (PyTypeObject *)PyExc_BaseExceptionGroup)) { + PyErr_SetString(PyExc_TypeError, + "derive must return an instance of BaseExceptionGroup"); + goto error; + } + + PyObject *tb = PyException_GetTraceback((PyObject*)orig); + if (tb) { + int res = PyException_SetTraceback(eg, tb); + Py_XDECREF(tb); + if (res == -1) { + goto error; + } + } + PyObject *context = PyException_GetContext((PyObject*)orig); + if (context) { + PyException_SetContext(eg, context); + } + PyObject *cause = PyException_GetCause((PyObject*)orig); + if (cause) { + PyException_SetCause(eg, cause); + } + return eg; +error: + Py_XDECREF(eg); + return NULL; +} + +enum _exceptiongroup_split_matcher_type { + /* Exception type or tuple of thereof */ + EXCEPTION_GROUP_MATCH_BY_TYPE = 0, + /* A PyFunction returning True for matching exceptions */ + EXCEPTION_GROUP_MATCH_BY_PREDICATE = 1, + /* An instance or container thereof, checked with equality */ + EXCEPTION_GROUP_MATCH_INSTANCES = 2 +}; + +struct _exceptiongroup_matcher { + enum _exceptiongroup_split_matcher_type type; + PyObject *value; +}; + +static int _set_matcher_type( + struct _exceptiongroup_matcher *matcher) +{ + /* the python API supports only BY_TYPE and BY_PREDICATE */ + if (PyExceptionClass_Check(matcher->value) || + PyTuple_CheckExact(matcher->value)) { + matcher->type = EXCEPTION_GROUP_MATCH_BY_TYPE; + return 0; + } + else if (PyFunction_Check(matcher->value)) { + matcher->type = EXCEPTION_GROUP_MATCH_BY_PREDICATE; + return 0; + } + PyErr_SetString( + PyExc_TypeError, + "expected a function, exception type or tuple of exception types"); + return -1; +} + +static int exceptiongroup_split_check_match( + PyObject *exc, const struct _exceptiongroup_matcher *matcher) +{ + switch (matcher->type) { + case EXCEPTION_GROUP_MATCH_BY_TYPE: { + return PyErr_GivenExceptionMatches(exc, matcher->value); + } + case EXCEPTION_GROUP_MATCH_BY_PREDICATE: { + PyObject *exc_matches = PyObject_CallOneArg(matcher->value, exc); + if (exc_matches == NULL) { + return -1; + } + int is_true = PyObject_IsTrue(exc_matches); + Py_DECREF(exc_matches); + return is_true; + } + case EXCEPTION_GROUP_MATCH_INSTANCES: { + if (PySequence_Check(matcher->value)) { + return PySequence_Contains(matcher->value, exc); + } + else { + return matcher->value == exc; + } + } + } + return 0; +} + +static PyObject * +exceptiongroup_split_recursive(PyObject *exc, const struct _exceptiongroup_matcher* matcher, int construct_rest) +{ + int is_match = exceptiongroup_split_check_match(exc, matcher); + if (is_match < 0) { + return NULL; + } + + if (is_match) { + /* Full match */ + return PyTuple_Pack(2, exc, Py_None); + } + else if (!PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_BaseExceptionGroup)) { + /* Leaf exception and no match */ + return PyTuple_Pack( + 2, Py_None, construct_rest ? (PyObject*)exc : Py_None); + } + else { + /* Partial match */ + PyBaseExceptionGroupObject *eg = _PyBaseExceptionGroupObject_cast(exc); + PyObject *match_list = NULL; + PyObject *rest_list = NULL; + PyObject *match_exc = NULL; + PyObject *rest_exc = NULL; + PyObject *result = NULL; + + Py_ssize_t num_excs = PySequence_Length(eg->excs); + if (num_excs < 0) { + goto done; + } + match_list = PyList_New(0); + if (!match_list) { + goto done; + } + if (construct_rest) { + rest_list = PyList_New(0); + if (!rest_list) { + goto done; + } + } + /* recursive calls */ + for (Py_ssize_t i = 0; i < num_excs; i++) { + PyObject *e = PySequence_GetItem(eg->excs, i); + if (!e) { + goto done; + } + PyObject *rec = exceptiongroup_split_recursive( + e, matcher, construct_rest); + Py_DECREF(e); + if (!rec) { + goto done; + } + if (!PyTuple_CheckExact(rec) || PyTuple_GET_SIZE(rec) != 2) { + PyErr_SetString(PyExc_RuntimeError, + "Internal error: invalid value"); + Py_DECREF(rec); + goto done; + } + int res = 0; + PyObject *e_match = PyTuple_GET_ITEM(rec, 0); + if (e_match != Py_None) { + res += PyList_Append(match_list, e_match); + } + PyObject *e_rest = PyTuple_GET_ITEM(rec, 1); + if (e_rest != Py_None) { + res += PyList_Append(rest_list, e_rest); + } + Py_DECREF(rec); + if (res < 0) { + goto done; + } + } + + /* construct result */ + match_exc = exceptiongroup_subset(eg, match_list); + if (!match_exc) { + goto done; + } + + if (construct_rest) { + rest_exc = exceptiongroup_subset(eg, rest_list); + if (!rest_exc) { + goto done; + } + } + else { + rest_exc = Py_NewRef(Py_None); + } + result = PyTuple_Pack(2, match_exc, rest_exc); + done: + Py_XDECREF(match_exc); + Py_XDECREF(rest_exc); + Py_XDECREF(match_list); + Py_XDECREF(rest_list); + return result; + } +} + +static PyObject * +BaseExceptionGroup_split(PyObject *self, PyObject *args) +{ + PyObject *matcher_value = NULL; + if (!PyArg_UnpackTuple(args, "split", 1, 1, &matcher_value)) { + return NULL; + } + + struct _exceptiongroup_matcher matcher; + matcher.value = matcher_value; + if (_set_matcher_type(&matcher) == -1) { + return NULL; + } + return exceptiongroup_split_recursive( + self, &matcher, 1 /* with_construct_rest */); +} + +static PyObject * +BaseExceptionGroup_subgroup(PyObject *self, PyObject *args) +{ + PyObject *matcher_value = NULL; + if (!PyArg_UnpackTuple(args, "subgroup", 1, 1, &matcher_value)) { + return NULL; + } + + struct _exceptiongroup_matcher matcher; + matcher.value = matcher_value; + if (_set_matcher_type(&matcher) == -1) { + return NULL; + } + PyObject *ret = exceptiongroup_split_recursive( + self, &matcher, 0 /* without construct_rest */); + + if (!ret) { + return NULL; + } + if (!PyTuple_CheckExact(ret)) { + PyErr_SetString(PyExc_RuntimeError, + "Internal error: expected a 2-tuple"); + Py_DECREF(ret); + return NULL; + } + PyObject *match = Py_NewRef(PyTuple_GetItem(ret, 0)); + Py_DECREF(ret); + return match; +} + +static PyMemberDef BaseExceptionGroup_members[] = { + {"message", T_OBJECT, offsetof(PyBaseExceptionGroupObject, msg), READONLY, + PyDoc_STR("exception message")}, + {"exceptions", T_OBJECT, offsetof(PyBaseExceptionGroupObject, excs), READONLY, + PyDoc_STR("nested exceptions")}, + {NULL} /* Sentinel */ +}; + +static PyMethodDef BaseExceptionGroup_methods[] = { + {"derive", (PyCFunction)BaseExceptionGroup_derive, METH_VARARGS}, + {"split", (PyCFunction)BaseExceptionGroup_split, METH_VARARGS}, + {"subgroup", (PyCFunction)BaseExceptionGroup_subgroup, METH_VARARGS}, + {NULL} +}; + +ComplexExtendsException(PyExc_BaseException, BaseExceptionGroup, + BaseExceptionGroup, BaseExceptionGroup_new /* new */, + BaseExceptionGroup_methods, BaseExceptionGroup_members, + 0 /* getset */, BaseExceptionGroup_str, + "A combination of multiple unrelated exceptions."); + +/* + * ExceptionGroup extends BaseExceptionGroup, Exception + */ +static PyObject* +create_exception_group_class(void) { + PyObject *bases = PyTuple_Pack( + 2, PyExc_BaseExceptionGroup, PyExc_Exception); + if (bases == NULL) { + return NULL; + } + + PyExc_ExceptionGroup = PyErr_NewException( + "builtins.ExceptionGroup", bases, NULL); + if (PyExc_ExceptionGroup == NULL) { + return NULL; + } + return PyExc_ExceptionGroup; +} + /* * KeyboardInterrupt extends BaseException */ @@ -2671,6 +3119,7 @@ _PyExc_Init(PyInterpreterState *interp) } while (0) PRE_INIT(BaseException); + PRE_INIT(BaseExceptionGroup); PRE_INIT(Exception); PRE_INIT(TypeError); PRE_INIT(StopAsyncIteration); @@ -2805,8 +3254,14 @@ _PyBuiltins_AddExceptions(PyObject *bltinmod) return _PyStatus_ERR("exceptions bootstrapping error."); } + if (!create_exception_group_class()) { + return _PyStatus_ERR("exceptions bootstrapping error."); + } + POST_INIT(BaseException); POST_INIT(Exception); + POST_INIT(BaseExceptionGroup); + POST_INIT(ExceptionGroup); POST_INIT(TypeError); POST_INIT(StopAsyncIteration); POST_INIT(StopIteration); diff --git a/PC/python3dll.c b/PC/python3dll.c index d9e6fd3e7ca7cb..41c233f046cbdf 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -754,6 +754,7 @@ EXPORT_DATA(PyExc_ArithmeticError) EXPORT_DATA(PyExc_AssertionError) EXPORT_DATA(PyExc_AttributeError) EXPORT_DATA(PyExc_BaseException) +EXPORT_DATA(PyExc_BaseExceptionGroup) EXPORT_DATA(PyExc_BlockingIOError) EXPORT_DATA(PyExc_BrokenPipeError) EXPORT_DATA(PyExc_BufferError) @@ -768,6 +769,7 @@ EXPORT_DATA(PyExc_EncodingWarning) EXPORT_DATA(PyExc_EnvironmentError) EXPORT_DATA(PyExc_EOFError) EXPORT_DATA(PyExc_Exception) +EXPORT_DATA(PyExc_ExceptionGroup) EXPORT_DATA(PyExc_FileExistsError) EXPORT_DATA(PyExc_FileNotFoundError) EXPORT_DATA(PyExc_FloatingPointError) From 0ac0dfdce4d3e886d82f50e1ff17e96644705e5c Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sun, 26 Sep 2021 18:18:52 +0000 Subject: [PATCH 02/40] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst new file mode 100644 index 00000000000000..0fe7720094381a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst @@ -0,0 +1 @@ +Implement PEP 654: Add :class:`ExceptionGroup` and :class:`BaseExceptionGroup`. \ No newline at end of file From b7ee952a8ec6377fd6f0649a54f77da178c94bb3 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Sep 2021 11:12:29 +0100 Subject: [PATCH 03/40] rename exceptions_added_in_python_3 --> exceptions_not_in_python_2 --- Lib/test/test_pickle.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 412949134986ef..f0becd5a547086 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -483,8 +483,10 @@ def test_exceptions(self): self.assertEqual(reverse_mapping('builtins', 'OSError'), ('exceptions', 'OSError')) - exceptions_added_in_python_3 = [ - 'BaseExceptionGroup', 'ExceptionGroup'] + exceptions_not_in_python_2 = [ + 'BaseExceptionGroup', # Added in 3.11 + 'ExceptionGroup' # Added in 3.11 + ] for name, exc in get_exceptions(builtins): with self.subTest(name): if exc in (BlockingIOError, @@ -501,7 +503,7 @@ def test_exceptions(self): ('exceptions', 'ImportError')) self.assertEqual(mapping('exceptions', name), ('exceptions', name)) - elif not name in exceptions_added_in_python_3: + elif name not in exceptions_not_in_python_2: self.assertEqual(reverse_mapping('builtins', name), ('exceptions', name)) self.assertEqual(mapping('exceptions', name), From a57f8839449c39686f48c30ddddb716d5fe3c464 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Sep 2021 11:42:36 +0100 Subject: [PATCH 04/40] Use PyArg_ParseTuple's format string to check for type of EG message --- Lib/test/test_exception_group.py | 2 +- Objects/exceptions.c | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 0c11c52e642734..21877ae12d4920 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -22,7 +22,7 @@ def test_bad_EG_construction__too_many_args(self): _ = ExceptionGroup('eg', [ValueError('too')], [TypeError('many')]) def test_bad_EG_construction__bad_message(self): - MSG = 'Expected a message' + MSG = 'argument 1 must be str, not ' with self.assertRaisesRegex(TypeError, MSG): _ = ExceptionGroup(ValueError(12), SyntaxError('bad syntax')) with self.assertRaisesRegex(TypeError, MSG): diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 0f826a4c5e01ea..a0ed5dd8832d23 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -649,11 +649,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *msg = NULL; PyObject *excs = NULL; - if (!PyArg_ParseTuple(args, "OO", &msg, &excs)) { - return NULL; - } - if (!PyUnicode_CheckExact(msg)) { - PyErr_SetString(PyExc_TypeError, "Expected a message"); + if (!PyArg_ParseTuple(args, "UO", &msg, &excs)) { return NULL; } Py_ssize_t numexcs = -1; From 62a7b3c48875a920160c7f374136ff9423dba1a1 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Sep 2021 12:57:16 +0100 Subject: [PATCH 05/40] improve error messages for second constructor arg --- Lib/test/test_exception_group.py | 4 ++-- Objects/exceptions.c | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 21877ae12d4920..92278b11fd95fd 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -29,7 +29,7 @@ def test_bad_EG_construction__bad_message(self): _ = ExceptionGroup(None, [ValueError(12)]) def test_bad_EG_construction__bad_excs_sequence(self): - MSG = 'Expected a non-empty sequence' + MSG = 'argument 2 must be a non-empty sequence' with self.assertRaisesRegex(TypeError, MSG): _ = ExceptionGroup('errors not sequence', {ValueError(42)}) with self.assertRaisesRegex(TypeError, MSG): @@ -38,7 +38,7 @@ def test_bad_EG_construction__bad_excs_sequence(self): _ = ExceptionGroup("eg", None) def test_bad_EG_construction__nested_non_exceptions(self): - MSG = 'Nested exceptions must derive from BaseException' + MSG = 'Item [0-9]+ of argument 2 is not an exception' with self.assertRaisesRegex(TypeError, MSG): _ = ExceptionGroup('bad error', ["not an exception"]) with self.assertRaisesRegex(TypeError, MSG): diff --git a/Objects/exceptions.c b/Objects/exceptions.c index a0ed5dd8832d23..df8efcb5c47927 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -657,7 +657,8 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) numexcs = PySequence_Length(excs); } if (numexcs <= 0) { - PyErr_SetString(PyExc_TypeError, "Expected a non-empty sequence"); + PyErr_SetString(PyExc_TypeError, + "argument 2 must be a non-empty sequence"); return NULL; } @@ -668,8 +669,9 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } if (!PyExceptionInstance_Check(exc)) { - PyErr_SetString(PyExc_TypeError, - "Nested exceptions must derive from BaseException"); + PyErr_Format( + PyExc_TypeError, + "Item %d of argument 2 is not an exception", i); Py_DECREF(exc); return NULL; } From 235504bc958f438c1e8e4be7ecfc099b6d37ef41 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Sep 2021 13:17:24 +0100 Subject: [PATCH 06/40] nested exception must be instance not type --- Lib/test/test_exception_group.py | 4 +++- Objects/exceptions.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 92278b11fd95fd..33f318e3c26e60 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -38,7 +38,9 @@ def test_bad_EG_construction__bad_excs_sequence(self): _ = ExceptionGroup("eg", None) def test_bad_EG_construction__nested_non_exceptions(self): - MSG = 'Item [0-9]+ of argument 2 is not an exception' + MSG = 'Item [0-9]+ of argument 2 is not an exception instance' + with self.assertRaisesRegex(TypeError, MSG): + _ = ExceptionGroup('expect instance, not type', [ValueError]); with self.assertRaisesRegex(TypeError, MSG): _ = ExceptionGroup('bad error', ["not an exception"]) with self.assertRaisesRegex(TypeError, MSG): diff --git a/Objects/exceptions.c b/Objects/exceptions.c index df8efcb5c47927..7ae365020988a9 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -671,7 +671,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!PyExceptionInstance_Check(exc)) { PyErr_Format( PyExc_TypeError, - "Item %d of argument 2 is not an exception", i); + "Item %d of argument 2 is not an exception instance", i); Py_DECREF(exc); return NULL; } From b7889df157802f592a583bcda2946c4df7785fd7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Sep 2021 16:01:44 +0100 Subject: [PATCH 07/40] tweak test --- Lib/test/test_doctest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 603eb432567509..8423cafa8c7966 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -668,7 +668,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 820 < len(tests) < 845 # approximate number of objects with docstrings + >>> 825 < len(tests) < 845 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests From 763a387fabd66ac63dcc4a120a7a044abb7a6139 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 27 Sep 2021 19:07:29 +0100 Subject: [PATCH 08/40] expect msg to always be valid --- Objects/exceptions.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 7ae365020988a9..0e05833486acc4 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -752,13 +752,9 @@ BaseExceptionGroup_traverse(PyBaseExceptionGroupObject *self, visitproc visit, v static PyObject * BaseExceptionGroup_str(PyBaseExceptionGroupObject *self) { - if (self->msg && PyUnicode_CheckExact(self->msg)) { - Py_INCREF(self->msg); - return self->msg; - } - else { - return BaseException_str((PyBaseExceptionObject *)self); - } + assert(self->msg); + assert(PyUnicode_Check(self->msg)); + return Py_NewRef(self->msg); } static PyObject * From 8912054607c3cc83da8517b2ee56c5b250d8a16c Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 1 Oct 2021 16:33:55 +0100 Subject: [PATCH 09/40] make [Base]ExceptionGroup generic types --- Lib/test/test_exception_group.py | 5 +++++ Objects/exceptions.c | 2 ++ 2 files changed, 7 insertions(+) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 33f318e3c26e60..943df7b5f4a0c6 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -10,6 +10,11 @@ def test_exception_group_types(self): self.assertTrue(issubclass(ExceptionGroup, BaseExceptionGroup)) self.assertTrue(issubclass(BaseExceptionGroup, BaseException)) + def test_exception_group_is_generic_type(self): + E = OSError + self.assertIsInstance(ExceptionGroup[E], types.GenericAlias) + self.assertIsInstance(BaseExceptionGroup[E], types.GenericAlias) + class BadConstructorArgs(unittest.TestCase): def test_bad_EG_construction__too_many_args(self): diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 0e05833486acc4..1834df8ff6acd6 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1040,6 +1040,8 @@ static PyMemberDef BaseExceptionGroup_members[] = { }; static PyMethodDef BaseExceptionGroup_methods[] = { + {"__class_getitem__", (PyCFunction)Py_GenericAlias, + METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {"derive", (PyCFunction)BaseExceptionGroup_derive, METH_VARARGS}, {"split", (PyCFunction)BaseExceptionGroup_split, METH_VARARGS}, {"subgroup", (PyCFunction)BaseExceptionGroup_subgroup, METH_VARARGS}, From 41d479960fd937c9b4c33fab0471b676f5ea1e69 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 5 Oct 2021 13:19:53 +0100 Subject: [PATCH 10/40] tidy up test_pickle --- Lib/test/test_pickle.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index f0becd5a547086..057af21e71fe47 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -483,17 +483,15 @@ def test_exceptions(self): self.assertEqual(reverse_mapping('builtins', 'OSError'), ('exceptions', 'OSError')) - exceptions_not_in_python_2 = [ - 'BaseExceptionGroup', # Added in 3.11 - 'ExceptionGroup' # Added in 3.11 - ] for name, exc in get_exceptions(builtins): with self.subTest(name): if exc in (BlockingIOError, ResourceWarning, StopAsyncIteration, RecursionError, - EncodingWarning): + EncodingWarning, + BaseExceptionGroup, + ExceptionGroup): continue if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), @@ -503,7 +501,7 @@ def test_exceptions(self): ('exceptions', 'ImportError')) self.assertEqual(mapping('exceptions', name), ('exceptions', name)) - elif name not in exceptions_not_in_python_2: + else: self.assertEqual(reverse_mapping('builtins', name), ('exceptions', name)) self.assertEqual(mapping('exceptions', name), From 488ad7c8a09ffe4f2d1f50f306852034a055c3b2 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 5 Oct 2021 14:33:12 +0100 Subject: [PATCH 11/40] add test that exception group is not subscriptable (generic type) --- Lib/test/test_exception_group.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 943df7b5f4a0c6..153bcbfa71a414 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -10,6 +10,10 @@ def test_exception_group_types(self): self.assertTrue(issubclass(ExceptionGroup, BaseExceptionGroup)) self.assertTrue(issubclass(BaseExceptionGroup, BaseException)) + def test_exception_is_not_generic_type(self): + with self.assertRaises(TypeError): + Exception[OSError] + def test_exception_group_is_generic_type(self): E = OSError self.assertIsInstance(ExceptionGroup[E], types.GenericAlias) From 1de7ca5972df560267276b58bf27f3b62608de99 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 7 Oct 2021 11:42:11 +0100 Subject: [PATCH 12/40] implement suggestions from Erlend --- Objects/exceptions.c | 161 ++++++++++++++++++++++--------------------- 1 file changed, 81 insertions(+), 80 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 1834df8ff6acd6..01047cd3b9e9ef 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -806,22 +806,22 @@ exceptiongroup_subset(PyBaseExceptionGroupObject *orig, PyObject *excs) PyObject *tb = PyException_GetTraceback((PyObject*)orig); if (tb) { int res = PyException_SetTraceback(eg, tb); - Py_XDECREF(tb); + Py_DECREF(tb); if (res == -1) { goto error; } } PyObject *context = PyException_GetContext((PyObject*)orig); if (context) { - PyException_SetContext(eg, context); + PyException_SetContext(eg, context); /* steals a ref */ } PyObject *cause = PyException_GetCause((PyObject*)orig); if (cause) { - PyException_SetCause(eg, cause); + PyException_SetCause(eg, cause); /* steals a ref */ } return eg; error: - Py_XDECREF(eg); + Py_DECREF(eg); return NULL; } @@ -839,8 +839,8 @@ struct _exceptiongroup_matcher { PyObject *value; }; -static int _set_matcher_type( - struct _exceptiongroup_matcher *matcher) +static int +_set_matcher_type(struct _exceptiongroup_matcher *matcher) { /* the python API supports only BY_TYPE and BY_PREDICATE */ if (PyExceptionClass_Check(matcher->value) || @@ -858,8 +858,9 @@ static int _set_matcher_type( return -1; } -static int exceptiongroup_split_check_match( - PyObject *exc, const struct _exceptiongroup_matcher *matcher) +static int +exceptiongroup_split_check_match(PyObject *exc, + const struct _exceptiongroup_matcher *matcher) { switch (matcher->type) { case EXCEPTION_GROUP_MATCH_BY_TYPE: { @@ -878,16 +879,16 @@ static int exceptiongroup_split_check_match( if (PySequence_Check(matcher->value)) { return PySequence_Contains(matcher->value, exc); } - else { - return matcher->value == exc; - } + return matcher->value == exc; } } return 0; } static PyObject * -exceptiongroup_split_recursive(PyObject *exc, const struct _exceptiongroup_matcher* matcher, int construct_rest) +exceptiongroup_split_recursive(PyObject *exc, + const struct _exceptiongroup_matcher* matcher, + int construct_rest) { int is_match = exceptiongroup_split_check_match(exc, matcher); if (is_match < 0) { @@ -903,85 +904,85 @@ exceptiongroup_split_recursive(PyObject *exc, const struct _exceptiongroup_match return PyTuple_Pack( 2, Py_None, construct_rest ? (PyObject*)exc : Py_None); } - else { - /* Partial match */ - PyBaseExceptionGroupObject *eg = _PyBaseExceptionGroupObject_cast(exc); - PyObject *match_list = NULL; - PyObject *rest_list = NULL; - PyObject *match_exc = NULL; - PyObject *rest_exc = NULL; - PyObject *result = NULL; - - Py_ssize_t num_excs = PySequence_Length(eg->excs); - if (num_excs < 0) { + + /* Partial match */ + PyBaseExceptionGroupObject *eg = _PyBaseExceptionGroupObject_cast(exc); + PyObject *match_list = NULL; + PyObject *rest_list = NULL; + PyObject *match_exc = NULL; + PyObject *rest_exc = NULL; + PyObject *result = NULL; + + Py_ssize_t num_excs = PySequence_Length(eg->excs); + if (num_excs < 0) { + goto done; + } + assert(num_excs > 0); + match_list = PyList_New(0); + if (!match_list) { + goto done; + } + if (construct_rest) { + rest_list = PyList_New(0); + if (!rest_list) { goto done; } - match_list = PyList_New(0); - if (!match_list) { + } + /* recursive calls */ + for (Py_ssize_t i = 0; i < num_excs; i++) { + PyObject *e = PySequence_GetItem(eg->excs, i); + if (!e) { goto done; } - if (construct_rest) { - rest_list = PyList_New(0); - if (!rest_list) { - goto done; - } + PyObject *rec = exceptiongroup_split_recursive( + e, matcher, construct_rest); + Py_DECREF(e); + if (!rec) { + goto done; } - /* recursive calls */ - for (Py_ssize_t i = 0; i < num_excs; i++) { - PyObject *e = PySequence_GetItem(eg->excs, i); - if (!e) { - goto done; - } - PyObject *rec = exceptiongroup_split_recursive( - e, matcher, construct_rest); - Py_DECREF(e); - if (!rec) { - goto done; - } - if (!PyTuple_CheckExact(rec) || PyTuple_GET_SIZE(rec) != 2) { - PyErr_SetString(PyExc_RuntimeError, - "Internal error: invalid value"); - Py_DECREF(rec); - goto done; - } - int res = 0; - PyObject *e_match = PyTuple_GET_ITEM(rec, 0); - if (e_match != Py_None) { - res += PyList_Append(match_list, e_match); - } - PyObject *e_rest = PyTuple_GET_ITEM(rec, 1); - if (e_rest != Py_None) { - res += PyList_Append(rest_list, e_rest); - } + if (!PyTuple_CheckExact(rec) || PyTuple_GET_SIZE(rec) != 2) { + PyErr_SetString(PyExc_RuntimeError, + "Internal error: invalid value"); Py_DECREF(rec); - if (res < 0) { - goto done; - } + goto done; } - - /* construct result */ - match_exc = exceptiongroup_subset(eg, match_list); - if (!match_exc) { + int res = 0; + PyObject *e_match = PyTuple_GET_ITEM(rec, 0); + if (e_match != Py_None) { + res += PyList_Append(match_list, e_match); + } + PyObject *e_rest = PyTuple_GET_ITEM(rec, 1); + if (e_rest != Py_None) { + res += PyList_Append(rest_list, e_rest); + } + Py_DECREF(rec); + if (res < 0) { goto done; } + } - if (construct_rest) { - rest_exc = exceptiongroup_subset(eg, rest_list); - if (!rest_exc) { - goto done; - } - } - else { - rest_exc = Py_NewRef(Py_None); + /* construct result */ + match_exc = exceptiongroup_subset(eg, match_list); + if (!match_exc) { + goto done; + } + + if (construct_rest) { + rest_exc = exceptiongroup_subset(eg, rest_list); + if (!rest_exc) { + goto done; } - result = PyTuple_Pack(2, match_exc, rest_exc); - done: - Py_XDECREF(match_exc); - Py_XDECREF(rest_exc); - Py_XDECREF(match_list); - Py_XDECREF(rest_list); - return result; } + else { + rest_exc = Py_NewRef(Py_None); + } + result = PyTuple_Pack(2, match_exc, rest_exc); +done: + Py_XDECREF(match_exc); + Py_XDECREF(rest_exc); + Py_XDECREF(match_list); + Py_XDECREF(rest_list); + return result; } static PyObject * @@ -1026,7 +1027,7 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *args) Py_DECREF(ret); return NULL; } - PyObject *match = Py_NewRef(PyTuple_GetItem(ret, 0)); + PyObject *match = Py_NewRef(PyTuple_GET_ITEM(ret, 0)); Py_DECREF(ret); return match; } From 1147f44cb3e6869de95ebac6d28188f3fb60f775 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 7 Oct 2021 11:49:45 +0100 Subject: [PATCH 13/40] add comment --- Objects/exceptions.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 01047cd3b9e9ef..0bb1e6e54352b5 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -917,7 +917,7 @@ exceptiongroup_split_recursive(PyObject *exc, if (num_excs < 0) { goto done; } - assert(num_excs > 0); + assert(num_excs > 0); /* checked in constructor, and excs is read-only */ match_list = PyList_New(0); if (!match_list) { goto done; From c71aff7e1d5ec1d27cc1e24432deee0b8521c146 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 16 Oct 2021 21:47:43 +0100 Subject: [PATCH 14/40] part of updates re Guido's review of the tests --- Lib/test/test_exception_group.py | 67 ++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 153bcbfa71a414..889235f3c8078b 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -24,37 +24,34 @@ class BadConstructorArgs(unittest.TestCase): def test_bad_EG_construction__too_many_args(self): MSG = 'function takes exactly 2 arguments' with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup('no errors') + ExceptionGroup('no errors') with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup([ValueError('no msg')]) + ExceptionGroup([ValueError('no msg')]) with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup('eg', [ValueError('too')], [TypeError('many')]) + ExceptionGroup('eg', [ValueError('too')], [TypeError('many')]) def test_bad_EG_construction__bad_message(self): MSG = 'argument 1 must be str, not ' with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup(ValueError(12), SyntaxError('bad syntax')) + ExceptionGroup(ValueError(12), SyntaxError('bad syntax')) with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup(None, [ValueError(12)]) + ExceptionGroup(None, [ValueError(12)]) def test_bad_EG_construction__bad_excs_sequence(self): MSG = 'argument 2 must be a non-empty sequence' with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup('errors not sequence', {ValueError(42)}) + ExceptionGroup('errors not sequence', {ValueError(42)}) with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup("eg", []) + ExceptionGroup("eg", []) with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup("eg", None) + ExceptionGroup("eg", None) def test_bad_EG_construction__nested_non_exceptions(self): MSG = 'Item [0-9]+ of argument 2 is not an exception instance' with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup('expect instance, not type', [ValueError]); + ExceptionGroup('expect instance, not type', [ValueError]); with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup('bad error', ["not an exception"]) - with self.assertRaisesRegex(TypeError, MSG): - _ = ExceptionGroup('eg1', - [ExceptionGroup('eg2', ["not an exception"])]) + ExceptionGroup('bad error', ["not an exception"]) class InstanceCreation(unittest.TestCase): @@ -108,6 +105,7 @@ def leaf_generator(exc, tbs=None): def tb_funcnames(tbs): + # tbs is a traceback or a list of tracebacks def names_list(tb): if tb is None: return None @@ -116,19 +114,27 @@ def names_list(tb): if isinstance(tbs, types.TracebackType): return names_list(tbs) - else: + else: # list of tracebacks return [names_list(tb) for tb in tbs] def tbs_for_leaf(leaf, eg): for e, tbs in leaf_generator(eg): - if e == leaf: + if e is leaf: return tbs class ExceptionGroupTestBase(unittest.TestCase): def assertMatchesTemplate(self, exc, exc_type, template): - """ Assert that the exception matches the template """ + """ Assert that the exception matches the template + + A template describes the shape of exc. If exc is a + leaf exception (i.e., not an exception group) then + template is an exception instance that has the + expected type and args value of exc. If exc is an + exception group, then template is a list of the + templates of its nested exceptions. + """ if exc_type is not None: self.assertIs(type(exc), exc_type) @@ -229,10 +235,10 @@ def test_basics_ExceptionGroup_fields_are_read_only(self): eg = self.eg self.assertEqual(type(eg.exceptions), tuple) - _ = eg.message + eg.message with self.assertRaises(AttributeError): eg.message = "new msg" - _ = eg.exceptions + eg.exceptions with self.assertRaises(AttributeError): eg.exceptions = [OSError('xyz')] @@ -247,7 +253,7 @@ def test_basics_leaf_generator(self): [self.root_tb_fnames, self.type_error_tb_fnames], [self.root_tb_fnames, self.value_error_tb_fnames]]) - def test_basics_tbs_for_leaf(self): + def test_basics__tbs_for_leaf__helper_works(self): eg = self.eg for e, tbs in leaf_generator(eg): self.assertSequenceEqual(tbs, tbs_for_leaf(e, eg)) @@ -318,11 +324,16 @@ def test_basics_split_by_type__no_match(self): def test_basics_split_by_type__match(self): eg = self.eg + VE = ValueError + TE = TypeError testcases = [ # (matcher, match_template, rest_template) - (ValueError, [ValueError(1), ValueError(2)], [TypeError(int)]), - (TypeError, [TypeError(int)], [ValueError(1), ValueError(2)]), - ((ValueError, TypeError), self.eg_template, None)] + (VE, [VE(1), VE(2)], [TE(int)]), + (TE, [TE(int)], [VE(1), VE(2)]), + ((VE, TE), self.eg_template, None), + ((OSError, VE), [VE(1), VE(2)], [TE(int)]), + ] + for match_type, match_template, rest_template in testcases: match, rest = eg.split(match_type) self.assertEqual(match.message, eg.message) @@ -332,6 +343,8 @@ def test_basics_split_by_type__match(self): self.assertEqual(rest.message, eg.message) self.assertMatchesTemplate( rest, ExceptionGroup, rest_template) + else: + self.assertIsNone(rest) def test_basics_split_by_predicate__passthrough(self): match, rest = self.eg.split(lambda e: True) @@ -345,11 +358,15 @@ def test_basics_split_by_predicate__no_match(self): def test_basics_split_by_predicate__match(self): eg = self.eg + VE = ValueError + TE = TypeError testcases = [ # (matcher, match_template, rest_template) - (ValueError, [ValueError(1), ValueError(2)], [TypeError(int)]), - (TypeError, [TypeError(int)], [ValueError(1), ValueError(2)]), - ((ValueError, TypeError), self.eg_template, None)] + (VE, [VE(1), VE(2)], [TE(int)]), + (TE, [TE(int)], [VE(1), VE(2)]), + ((VE, TE), self.eg_template, None), + ] + for match_type, match_template, rest_template in testcases: match, rest = eg.split(lambda e: isinstance(e, match_type)) self.assertEqual(match.message, eg.message) From d3ba995c25e674ea81f482cafe80591254768da0 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 17 Oct 2021 17:19:59 +0100 Subject: [PATCH 15/40] Part 1 of Gregory's review: better error messaages, public GC API, bool type, flag name (is_nonbase_exception) --- Lib/test/test_exception_group.py | 10 +++++----- Objects/exceptions.c | 29 +++++++++++++++++------------ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 889235f3c8078b..5f2d2d5f17f328 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -38,7 +38,7 @@ def test_bad_EG_construction__bad_message(self): ExceptionGroup(None, [ValueError(12)]) def test_bad_EG_construction__bad_excs_sequence(self): - MSG = 'argument 2 must be a non-empty sequence' + MSG = 'second argument \(exceptions\) must be a non-empty sequence' with self.assertRaisesRegex(TypeError, MSG): ExceptionGroup('errors not sequence', {ValueError(42)}) with self.assertRaisesRegex(TypeError, MSG): @@ -47,10 +47,10 @@ def test_bad_EG_construction__bad_excs_sequence(self): ExceptionGroup("eg", None) def test_bad_EG_construction__nested_non_exceptions(self): - MSG = 'Item [0-9]+ of argument 2 is not an exception instance' - with self.assertRaisesRegex(TypeError, MSG): - ExceptionGroup('expect instance, not type', [ValueError]); - with self.assertRaisesRegex(TypeError, MSG): + MSG = 'Item [0-9]+ of second argument \(exceptions\) is not an exception' + with self.assertRaisesRegex(ValueError, MSG): + ExceptionGroup('expect instance, not type', [OSError]); + with self.assertRaisesRegex(ValueError, MSG): ExceptionGroup('bad error', ["not an exception"]) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 0bb1e6e54352b5..a2fec23e7df3c4 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -6,6 +6,7 @@ #define PY_SSIZE_T_CLEAN #include +#include #include "pycore_initconfig.h" #include "pycore_object.h" #include "structmember.h" // PyMemberDef @@ -540,7 +541,7 @@ StopIteration_clear(PyStopIterationObject *self) static void StopIteration_dealloc(PyStopIterationObject *self) { - _PyObject_GC_UNTRACK(self); + PyObject_GC_UnTrack(self); StopIteration_clear(self); Py_TYPE(self)->tp_free((PyObject *)self); } @@ -657,12 +658,13 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) numexcs = PySequence_Length(excs); } if (numexcs <= 0) { - PyErr_SetString(PyExc_TypeError, - "argument 2 must be a non-empty sequence"); + PyErr_SetString( + PyExc_TypeError, + "second argument (exceptions) must be a non-empty sequence"); return NULL; } - int nested_base_exceptions = 0; + bool nested_base_exceptions = false; for (Py_ssize_t i = 0; i < numexcs; i++) { PyObject *exc = PySequence_GetItem(excs, i); if (!exc) { @@ -670,18 +672,19 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } if (!PyExceptionInstance_Check(exc)) { PyErr_Format( - PyExc_TypeError, - "Item %d of argument 2 is not an exception instance", i); + PyExc_ValueError, + "Item %d of second argument (exceptions) is not an exception", + i); Py_DECREF(exc); return NULL; } - int is_exception = PyObject_IsInstance(exc, PyExc_Exception); + int is_nonbase_exception = PyObject_IsInstance(exc, PyExc_Exception); Py_DECREF(exc); - if (is_exception < 0) { + if (is_nonbase_exception < 0) { return NULL; } - else if (is_exception == 0) { - nested_base_exceptions = 1; + else if (is_nonbase_exception == 0) { + nested_base_exceptions = true; } } @@ -742,7 +745,8 @@ BaseExceptionGroup_dealloc(PyBaseExceptionGroupObject *self) } static int -BaseExceptionGroup_traverse(PyBaseExceptionGroupObject *self, visitproc visit, void *arg) +BaseExceptionGroup_traverse(PyBaseExceptionGroupObject *self, + visitproc visit, void *arg) { Py_VISIT(self->msg); Py_VISIT(self->excs); @@ -758,7 +762,8 @@ BaseExceptionGroup_str(PyBaseExceptionGroupObject *self) } static PyObject * -BaseExceptionGroup_derive(PyObject *self_, PyObject *args) { +BaseExceptionGroup_derive(PyObject *self_, PyObject *args) +{ PyBaseExceptionGroupObject *self = _PyBaseExceptionGroupObject_cast(self_); PyObject *excs = NULL; if (!PyArg_ParseTuple(args, "O", &excs)) { From fe5d6110d15018299f1421944e4940306822d387 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 17 Oct 2021 21:21:30 +0100 Subject: [PATCH 16/40] Gregory's suggestion to convert excs to Tuple at the beginning of the constructor --- Lib/test/test_exception_group.py | 8 ++++--- Objects/exceptions.c | 41 ++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 5f2d2d5f17f328..b44491239055aa 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -38,14 +38,16 @@ def test_bad_EG_construction__bad_message(self): ExceptionGroup(None, [ValueError(12)]) def test_bad_EG_construction__bad_excs_sequence(self): - MSG = 'second argument \(exceptions\) must be a non-empty sequence' + MSG = 'second argument \(exceptions\) must be a sequence' with self.assertRaisesRegex(TypeError, MSG): ExceptionGroup('errors not sequence', {ValueError(42)}) - with self.assertRaisesRegex(TypeError, MSG): - ExceptionGroup("eg", []) with self.assertRaisesRegex(TypeError, MSG): ExceptionGroup("eg", None) + MSG = 'second argument \(exceptions\) must be a non-empty sequence' + with self.assertRaisesRegex(TypeError, MSG): + ExceptionGroup("eg", []) + def test_bad_EG_construction__nested_non_exceptions(self): MSG = 'Item [0-9]+ of second argument \(exceptions\) is not an exception' with self.assertRaisesRegex(ValueError, MSG): diff --git a/Objects/exceptions.c b/Objects/exceptions.c index a2fec23e7df3c4..88ccb3a2b229c1 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -653,35 +653,45 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!PyArg_ParseTuple(args, "UO", &msg, &excs)) { return NULL; } - Py_ssize_t numexcs = -1; - if (PySequence_Check(excs)) { - numexcs = PySequence_Length(excs); + + if (!PySequence_Check(excs)) { + PyErr_SetString( + PyExc_TypeError, + "second argument (exceptions) must be a sequence"); + return NULL; + } + + excs = PySequence_Tuple(excs); + if (!excs) { + return NULL; } + + /* We are now holding a ref to the excs tuple */ + + Py_ssize_t numexcs = PySequence_Length(excs); if (numexcs <= 0) { PyErr_SetString( PyExc_TypeError, "second argument (exceptions) must be a non-empty sequence"); - return NULL; + goto error; } bool nested_base_exceptions = false; for (Py_ssize_t i = 0; i < numexcs; i++) { - PyObject *exc = PySequence_GetItem(excs, i); + PyObject *exc = PyTuple_GET_ITEM(excs, i); if (!exc) { - return NULL; + goto error; } if (!PyExceptionInstance_Check(exc)) { PyErr_Format( PyExc_ValueError, "Item %d of second argument (exceptions) is not an exception", i); - Py_DECREF(exc); - return NULL; + goto error; } int is_nonbase_exception = PyObject_IsInstance(exc, PyExc_Exception); - Py_DECREF(exc); if (is_nonbase_exception < 0) { - return NULL; + goto error; } else if (is_nonbase_exception == 0) { nested_base_exceptions = true; @@ -693,7 +703,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (nested_base_exceptions) { PyErr_SetString(PyExc_TypeError, "Cannot nest BaseExceptions in an ExceptionGroup"); - return NULL; + goto error; } } else if (cls == (PyTypeObject*)PyExc_BaseExceptionGroup) { @@ -710,9 +720,16 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyBaseExceptionGroupObject *self = _PyBaseExceptionGroupObject_cast(BaseException_new(cls, args, kwds)); + if (!self) { + goto error; + } + self->msg = Py_NewRef(msg); - self->excs = PySequence_Tuple(excs); + self->excs = excs; return (PyObject*)self; +error: + Py_DECREF(excs); + return NULL; } static int From ed343debb48d8c464cf6301a460d4895693f7896 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 17 Oct 2021 21:53:24 +0100 Subject: [PATCH 17/40] rename constructor args to match their python names --- Objects/exceptions.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 88ccb3a2b229c1..1e80e48d9f0cbe 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -647,28 +647,28 @@ _PyBaseExceptionGroupObject_cast(PyObject *exc) static PyObject * BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject *msg = NULL; - PyObject *excs = NULL; + PyObject *message = NULL; + PyObject *exceptions = NULL; - if (!PyArg_ParseTuple(args, "UO", &msg, &excs)) { + if (!PyArg_ParseTuple(args, "UO", &message, &exceptions)) { return NULL; } - if (!PySequence_Check(excs)) { + if (!PySequence_Check(exceptions)) { PyErr_SetString( PyExc_TypeError, "second argument (exceptions) must be a sequence"); return NULL; } - excs = PySequence_Tuple(excs); - if (!excs) { + exceptions = PySequence_Tuple(exceptions); + if (!exceptions) { return NULL; } - /* We are now holding a ref to the excs tuple */ + /* We are now holding a ref to the exceptions tuple */ - Py_ssize_t numexcs = PySequence_Length(excs); + Py_ssize_t numexcs = PySequence_Length(exceptions); if (numexcs <= 0) { PyErr_SetString( PyExc_TypeError, @@ -678,7 +678,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) bool nested_base_exceptions = false; for (Py_ssize_t i = 0; i < numexcs; i++) { - PyObject *exc = PyTuple_GET_ITEM(excs, i); + PyObject *exc = PyTuple_GET_ITEM(exceptions, i); if (!exc) { goto error; } @@ -724,11 +724,11 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - self->msg = Py_NewRef(msg); - self->excs = excs; + self->msg = Py_NewRef(message); + self->excs = exceptions; return (PyObject*)self; error: - Py_DECREF(excs); + Py_DECREF(exceptions); return NULL; } From 1af4e5323257591bbce4a2543a4de3f9f31ce5f0 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 17 Oct 2021 21:56:50 +0100 Subject: [PATCH 18/40] empty-exceptions error changed from TypeError to ValueError --- Lib/test/test_exception_group.py | 2 +- Objects/exceptions.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index b44491239055aa..f544d7ef309afa 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -45,7 +45,7 @@ def test_bad_EG_construction__bad_excs_sequence(self): ExceptionGroup("eg", None) MSG = 'second argument \(exceptions\) must be a non-empty sequence' - with self.assertRaisesRegex(TypeError, MSG): + with self.assertRaisesRegex(ValueError, MSG): ExceptionGroup("eg", []) def test_bad_EG_construction__nested_non_exceptions(self): diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 1e80e48d9f0cbe..407a2c022b7c4c 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -671,7 +671,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_ssize_t numexcs = PySequence_Length(exceptions); if (numexcs <= 0) { PyErr_SetString( - PyExc_TypeError, + PyExc_ValueError, "second argument (exceptions) must be a non-empty sequence"); goto error; } From bc57273122a25f8ba1e7c85e9595f7746a7148fc Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 18 Oct 2021 11:08:17 +0100 Subject: [PATCH 19/40] decref bases in create_exception_group_class --- Objects/exceptions.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 407a2c022b7c4c..486dfbaf724a0a 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1090,6 +1090,7 @@ create_exception_group_class(void) { PyExc_ExceptionGroup = PyErr_NewException( "builtins.ExceptionGroup", bases, NULL); + Py_DECREF(bases); if (PyExc_ExceptionGroup == NULL) { return NULL; } From a21c174bd73d67cd0b91365cfc3987c6af24e6fa Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 18 Oct 2021 11:09:44 +0100 Subject: [PATCH 20/40] remove redundant check --- Objects/exceptions.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 486dfbaf724a0a..b1b46350b5731b 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1091,9 +1091,6 @@ create_exception_group_class(void) { PyExc_ExceptionGroup = PyErr_NewException( "builtins.ExceptionGroup", bases, NULL); Py_DECREF(bases); - if (PyExc_ExceptionGroup == NULL) { - return NULL; - } return PyExc_ExceptionGroup; } From 0e462475ccac18e136c4f8599cba2d6ee89c44aa Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 18 Oct 2021 14:58:36 +0100 Subject: [PATCH 21/40] add _PyBaseExceptionGroup_Check --- Include/pyerrors.h | 2 ++ Objects/exceptions.c | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Include/pyerrors.h b/Include/pyerrors.h index b2693c969d7176..87ba2ee28599b8 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -60,6 +60,8 @@ PyAPI_FUNC(const char *) PyExceptionClass_Name(PyObject *); #define PyExceptionInstance_Class(x) ((PyObject*)Py_TYPE(x)) +#define _PyBaseExceptionGroup_Check(x) \ + PyObject_TypeCheck(x, (PyTypeObject *)PyExc_BaseExceptionGroup) /* Predefined exceptions */ diff --git a/Objects/exceptions.c b/Objects/exceptions.c index b1b46350b5731b..36d5a941ad8fd6 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -640,7 +640,7 @@ PyObject *PyExc_ExceptionGroup; static inline PyBaseExceptionGroupObject* _PyBaseExceptionGroupObject_cast(PyObject *exc) { - assert(PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_BaseExceptionGroup)); + assert(_PyBaseExceptionGroup_Check(exc)); return (PyBaseExceptionGroupObject *)exc; } @@ -819,7 +819,7 @@ exceptiongroup_subset(PyBaseExceptionGroupObject *orig, PyObject *excs) return NULL; } - if (!PyObject_TypeCheck(eg, (PyTypeObject *)PyExc_BaseExceptionGroup)) { + if (!_PyBaseExceptionGroup_Check(eg)) { PyErr_SetString(PyExc_TypeError, "derive must return an instance of BaseExceptionGroup"); goto error; @@ -921,7 +921,7 @@ exceptiongroup_split_recursive(PyObject *exc, /* Full match */ return PyTuple_Pack(2, exc, Py_None); } - else if (!PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_BaseExceptionGroup)) { + else if (!_PyBaseExceptionGroup_Check(exc)) { /* Leaf exception and no match */ return PyTuple_Pack( 2, Py_None, construct_rest ? (PyObject*)exc : Py_None); From 84c52aedab2f21cfa47f2ec9927af521f0e7468f Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 18 Oct 2021 15:29:47 +0100 Subject: [PATCH 22/40] Fix error checking. Exception --> assertion --- Objects/exceptions.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 36d5a941ad8fd6..e621b48c052779 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -962,25 +962,22 @@ exceptiongroup_split_recursive(PyObject *exc, if (!rec) { goto done; } - if (!PyTuple_CheckExact(rec) || PyTuple_GET_SIZE(rec) != 2) { - PyErr_SetString(PyExc_RuntimeError, - "Internal error: invalid value"); - Py_DECREF(rec); - goto done; - } - int res = 0; + assert(PyTuple_CheckExact(rec) && PyTuple_GET_SIZE(rec) == 2); PyObject *e_match = PyTuple_GET_ITEM(rec, 0); if (e_match != Py_None) { - res += PyList_Append(match_list, e_match); + if (PyList_Append(match_list, e_match) < 0) { + Py_DECREF(rec); + goto done; + } } PyObject *e_rest = PyTuple_GET_ITEM(rec, 1); if (e_rest != Py_None) { - res += PyList_Append(rest_list, e_rest); + if (PyList_Append(rest_list, e_rest) < 0) { + Py_DECREF(rec); + goto done; + } } Py_DECREF(rec); - if (res < 0) { - goto done; - } } /* construct result */ From 1813827536a15c1e449b3978d8fe0f1c709764ab Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 18 Oct 2021 15:40:32 +0100 Subject: [PATCH 23/40] add assertions, convert runtime check to assertion --- Objects/exceptions.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index e621b48c052779..2021d05548c0a3 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -972,6 +972,8 @@ exceptiongroup_split_recursive(PyObject *exc, } PyObject *e_rest = PyTuple_GET_ITEM(rec, 1); if (e_rest != Py_None) { + assert(construct_rest); + assert(PyList_CheckExact(rest_list)); if (PyList_Append(rest_list, e_rest) < 0) { Py_DECREF(rec); goto done; @@ -1040,12 +1042,7 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *args) if (!ret) { return NULL; } - if (!PyTuple_CheckExact(ret)) { - PyErr_SetString(PyExc_RuntimeError, - "Internal error: expected a 2-tuple"); - Py_DECREF(ret); - return NULL; - } + assert(PyTuple_CheckExact(ret) && PyTuple_GET_SIZE(ret) == 2); PyObject *match = Py_NewRef(PyTuple_GET_ITEM(ret, 0)); Py_DECREF(ret); return match; From 3201ab906b63e1a0c391fa842aed924e5e45b6c1 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 18 Oct 2021 15:47:44 +0100 Subject: [PATCH 24/40] NULL is not error for GetContext/SetContext etc (no need to check) --- Objects/exceptions.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 2021d05548c0a3..e098e87bd3590d 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -833,14 +833,8 @@ exceptiongroup_subset(PyBaseExceptionGroupObject *orig, PyObject *excs) goto error; } } - PyObject *context = PyException_GetContext((PyObject*)orig); - if (context) { - PyException_SetContext(eg, context); /* steals a ref */ - } - PyObject *cause = PyException_GetCause((PyObject*)orig); - if (cause) { - PyException_SetCause(eg, cause); /* steals a ref */ - } + PyException_SetContext(eg, PyException_GetContext((PyObject*)orig)); + PyException_SetCause(eg, PyException_GetCause((PyObject*)orig)); return eg; error: Py_DECREF(eg); From 37d8df2b81fac6cfcfbc26be7af07a4597361ead Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 18 Oct 2021 16:09:17 +0100 Subject: [PATCH 25/40] cast orig only once in exceptiongroup_subset --- Objects/exceptions.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index e098e87bd3590d..c806dadf3bfade 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -797,7 +797,7 @@ BaseExceptionGroup_derive(PyObject *self_, PyObject *args) } static PyObject* -exceptiongroup_subset(PyBaseExceptionGroupObject *orig, PyObject *excs) +exceptiongroup_subset(PyBaseExceptionGroupObject *_orig, PyObject *excs) { /* Return an ExceptionGroup wrapping excs with metadata from orig. @@ -805,6 +805,7 @@ exceptiongroup_subset(PyBaseExceptionGroupObject *orig, PyObject *excs) so excs is the matching or non-matching sub-sequence of orig->excs (this function does not verify that it is a subsequence). */ + PyObject *orig = (PyObject *)_orig; Py_ssize_t num_excs = PySequence_Size(excs); if (num_excs < 0) { return NULL; @@ -814,7 +815,7 @@ exceptiongroup_subset(PyBaseExceptionGroupObject *orig, PyObject *excs) } PyObject *eg = PyObject_CallMethod( - (PyObject*)orig, "derive", "(O)", excs); + orig, "derive", "(O)", excs); if (!eg) { return NULL; } @@ -825,7 +826,7 @@ exceptiongroup_subset(PyBaseExceptionGroupObject *orig, PyObject *excs) goto error; } - PyObject *tb = PyException_GetTraceback((PyObject*)orig); + PyObject *tb = PyException_GetTraceback(orig); if (tb) { int res = PyException_SetTraceback(eg, tb); Py_DECREF(tb); @@ -833,8 +834,8 @@ exceptiongroup_subset(PyBaseExceptionGroupObject *orig, PyObject *excs) goto error; } } - PyException_SetContext(eg, PyException_GetContext((PyObject*)orig)); - PyException_SetCause(eg, PyException_GetCause((PyObject*)orig)); + PyException_SetContext(eg, PyException_GetContext(orig)); + PyException_SetCause(eg, PyException_GetCause(orig)); return eg; error: Py_DECREF(eg); From 0099c2448559a43ee2c3b6dc5770fd69039ff1c2 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 18 Oct 2021 16:49:45 +0100 Subject: [PATCH 26/40] assume (and assert) that eg->excs is a tuple, use Tuple apis --- Objects/exceptions.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index c806dadf3bfade..673d1dd339ae06 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -668,8 +668,8 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* We are now holding a ref to the exceptions tuple */ - Py_ssize_t numexcs = PySequence_Length(exceptions); - if (numexcs <= 0) { + Py_ssize_t numexcs = PyTuple_GET_SIZE(exceptions); + if (numexcs == 0) { PyErr_SetString( PyExc_ValueError, "second argument (exceptions) must be a non-empty sequence"); @@ -930,7 +930,8 @@ exceptiongroup_split_recursive(PyObject *exc, PyObject *rest_exc = NULL; PyObject *result = NULL; - Py_ssize_t num_excs = PySequence_Length(eg->excs); + assert(PyTuple_CheckExact(eg->excs)); + Py_ssize_t num_excs = PyTuple_Size(eg->excs); if (num_excs < 0) { goto done; } @@ -947,19 +948,17 @@ exceptiongroup_split_recursive(PyObject *exc, } /* recursive calls */ for (Py_ssize_t i = 0; i < num_excs; i++) { - PyObject *e = PySequence_GetItem(eg->excs, i); - if (!e) { - goto done; - } + PyObject *e = PyTuple_GET_ITEM(eg->excs, i); + assert(e); PyObject *rec = exceptiongroup_split_recursive( e, matcher, construct_rest); - Py_DECREF(e); if (!rec) { goto done; } assert(PyTuple_CheckExact(rec) && PyTuple_GET_SIZE(rec) == 2); PyObject *e_match = PyTuple_GET_ITEM(rec, 0); if (e_match != Py_None) { + assert(PyList_CheckExact(match_list)); if (PyList_Append(match_list, e_match) < 0) { Py_DECREF(rec); goto done; From bab699bd1611394735c1a10a17a15a3747673143 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 18 Oct 2021 17:16:47 +0100 Subject: [PATCH 27/40] no need for the fancy _exceptiongroup_matcher struct --- Objects/exceptions.c | 59 +++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 673d1dd339ae06..cc35013b48d4f5 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -842,31 +842,27 @@ exceptiongroup_subset(PyBaseExceptionGroupObject *_orig, PyObject *excs) return NULL; } -enum _exceptiongroup_split_matcher_type { +typedef enum { /* Exception type or tuple of thereof */ EXCEPTION_GROUP_MATCH_BY_TYPE = 0, /* A PyFunction returning True for matching exceptions */ EXCEPTION_GROUP_MATCH_BY_PREDICATE = 1, /* An instance or container thereof, checked with equality */ EXCEPTION_GROUP_MATCH_INSTANCES = 2 -}; - -struct _exceptiongroup_matcher { - enum _exceptiongroup_split_matcher_type type; - PyObject *value; -}; +} _exceptiongroup_split_matcher_type; static int -_set_matcher_type(struct _exceptiongroup_matcher *matcher) +_get_matcher_type(PyObject *value, + _exceptiongroup_split_matcher_type *type) { /* the python API supports only BY_TYPE and BY_PREDICATE */ - if (PyExceptionClass_Check(matcher->value) || - PyTuple_CheckExact(matcher->value)) { - matcher->type = EXCEPTION_GROUP_MATCH_BY_TYPE; + if (PyExceptionClass_Check(value) || + PyTuple_CheckExact(value)) { + *type = EXCEPTION_GROUP_MATCH_BY_TYPE; return 0; } - else if (PyFunction_Check(matcher->value)) { - matcher->type = EXCEPTION_GROUP_MATCH_BY_PREDICATE; + if (PyFunction_Check(value)) { + *type = EXCEPTION_GROUP_MATCH_BY_PREDICATE; return 0; } PyErr_SetString( @@ -877,14 +873,15 @@ _set_matcher_type(struct _exceptiongroup_matcher *matcher) static int exceptiongroup_split_check_match(PyObject *exc, - const struct _exceptiongroup_matcher *matcher) + _exceptiongroup_split_matcher_type matcher_type, + PyObject *matcher_value) { - switch (matcher->type) { + switch (matcher_type) { case EXCEPTION_GROUP_MATCH_BY_TYPE: { - return PyErr_GivenExceptionMatches(exc, matcher->value); + return PyErr_GivenExceptionMatches(exc, matcher_value); } case EXCEPTION_GROUP_MATCH_BY_PREDICATE: { - PyObject *exc_matches = PyObject_CallOneArg(matcher->value, exc); + PyObject *exc_matches = PyObject_CallOneArg(matcher_value, exc); if (exc_matches == NULL) { return -1; } @@ -893,10 +890,10 @@ exceptiongroup_split_check_match(PyObject *exc, return is_true; } case EXCEPTION_GROUP_MATCH_INSTANCES: { - if (PySequence_Check(matcher->value)) { - return PySequence_Contains(matcher->value, exc); + if (PySequence_Check(matcher_value)) { + return PySequence_Contains(matcher_value, exc); } - return matcher->value == exc; + return matcher_value == exc; } } return 0; @@ -904,10 +901,12 @@ exceptiongroup_split_check_match(PyObject *exc, static PyObject * exceptiongroup_split_recursive(PyObject *exc, - const struct _exceptiongroup_matcher* matcher, + _exceptiongroup_split_matcher_type matcher_type, + PyObject *matcher_value, int construct_rest) { - int is_match = exceptiongroup_split_check_match(exc, matcher); + int is_match = exceptiongroup_split_check_match( + exc, matcher_type, matcher_value); if (is_match < 0) { return NULL; } @@ -951,7 +950,7 @@ exceptiongroup_split_recursive(PyObject *exc, PyObject *e = PyTuple_GET_ITEM(eg->excs, i); assert(e); PyObject *rec = exceptiongroup_split_recursive( - e, matcher, construct_rest); + e, matcher_type, matcher_value, construct_rest); if (!rec) { goto done; } @@ -1008,13 +1007,12 @@ BaseExceptionGroup_split(PyObject *self, PyObject *args) return NULL; } - struct _exceptiongroup_matcher matcher; - matcher.value = matcher_value; - if (_set_matcher_type(&matcher) == -1) { + _exceptiongroup_split_matcher_type matcher_type; + if (_get_matcher_type(matcher_value, &matcher_type) == -1) { return NULL; } return exceptiongroup_split_recursive( - self, &matcher, 1 /* with_construct_rest */); + self, matcher_type, matcher_value, 1 /* with_construct_rest */); } static PyObject * @@ -1025,13 +1023,12 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *args) return NULL; } - struct _exceptiongroup_matcher matcher; - matcher.value = matcher_value; - if (_set_matcher_type(&matcher) == -1) { + _exceptiongroup_split_matcher_type matcher_type; + if (_get_matcher_type(matcher_value, &matcher_type) == -1) { return NULL; } PyObject *ret = exceptiongroup_split_recursive( - self, &matcher, 0 /* without construct_rest */); + self, matcher_type, matcher_value, 0 /* without construct_rest */); if (!ret) { return NULL; From 0603d31ebeaab8151e28058dead2089170d32119 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 19 Oct 2021 14:55:58 +0100 Subject: [PATCH 28/40] move ExceptionGroup type definition to interpreter state --- Include/internal/pycore_interp.h | 2 ++ Objects/exceptions.c | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 64ac3abe00fa01..c16f0a4b5e643e 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -205,6 +205,8 @@ struct _Py_exc_state { PyObject *errnomap; PyBaseExceptionObject *memerrors_freelist; int memerrors_numfree; + // The ExceptionGroup type + PyObject *PyExc_ExceptionGroup; }; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index cc35013b48d4f5..36c6b7075f560d 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -635,7 +635,6 @@ ComplexExtendsException(PyExc_BaseException, SystemExit, SystemExit, * ExceptionGroup extends BaseExceptionGroup and Exception */ -PyObject *PyExc_ExceptionGroup; static inline PyBaseExceptionGroupObject* _PyBaseExceptionGroupObject_cast(PyObject *exc) @@ -647,6 +646,9 @@ _PyBaseExceptionGroupObject_cast(PyObject *exc) static PyObject * BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { + struct _Py_exc_state *state = get_exc_state(); + PyObject *PyExc_ExceptionGroup = state->PyExc_ExceptionGroup; + PyObject *message = NULL; PyObject *exceptions = NULL; @@ -1067,16 +1069,20 @@ ComplexExtendsException(PyExc_BaseException, BaseExceptionGroup, */ static PyObject* create_exception_group_class(void) { + struct _Py_exc_state *state = get_exc_state(); + PyObject *bases = PyTuple_Pack( 2, PyExc_BaseExceptionGroup, PyExc_Exception); if (bases == NULL) { return NULL; } - PyExc_ExceptionGroup = PyErr_NewException( + assert(!state->PyExc_ExceptionGroup); + state->PyExc_ExceptionGroup = PyErr_NewException( "builtins.ExceptionGroup", bases, NULL); + Py_DECREF(bases); - return PyExc_ExceptionGroup; + return state->PyExc_ExceptionGroup; } /* @@ -3256,7 +3262,8 @@ _PyBuiltins_AddExceptions(PyObject *bltinmod) return _PyStatus_ERR("exceptions bootstrapping error."); } - if (!create_exception_group_class()) { + PyObject *PyExc_ExceptionGroup = create_exception_group_class(); + if (!PyExc_ExceptionGroup) { return _PyStatus_ERR("exceptions bootstrapping error."); } @@ -3348,6 +3355,7 @@ _PyExc_Fini(PyInterpreterState *interp) struct _Py_exc_state *state = &interp->exc_state; free_preallocated_memerrors(state); Py_CLEAR(state->errnomap); + Py_CLEAR(state->PyExc_ExceptionGroup); } /* Helper to do the equivalent of "raise X from Y" in C, but always using From db8d0b285f3243c5415b8765071f906b32bce90f Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 19 Oct 2021 15:46:55 +0100 Subject: [PATCH 29/40] clear ExceptionGroupType before interpreter state --- Include/internal/pycore_pylifecycle.h | 1 + Objects/exceptions.c | 12 +++++++++++- Python/pylifecycle.c | 2 ++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 4f12fef8d65466..53b94748b32e9e 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -93,6 +93,7 @@ extern void _PyAsyncGen_Fini(PyInterpreterState *interp); extern int _PySignal_Init(int install_signal_handlers); extern void _PySignal_Fini(void); +extern void _PyExc_ClearExceptionGroupType(PyInterpreterState *interp); extern void _PyExc_Fini(PyInterpreterState *interp); extern void _PyImport_Fini(void); extern void _PyImport_Fini2(void); diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 36c6b7075f560d..78f5126807b05d 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -720,6 +720,10 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* Do nothing - we don't interfere with subclasses */ } + if (!cls) { + /* Don't crash during interpreter shutdown */ + cls = (PyTypeObject*)PyExc_BaseExceptionGroup; + } PyBaseExceptionGroupObject *self = _PyBaseExceptionGroupObject_cast(BaseException_new(cls, args, kwds)); if (!self) { @@ -3349,13 +3353,19 @@ _PyBuiltins_AddExceptions(PyObject *bltinmod) #undef INIT_ALIAS } +void +_PyExc_ClearExceptionGroupType(PyInterpreterState *interp) +{ + struct _Py_exc_state *state = &interp->exc_state; + Py_CLEAR(state->PyExc_ExceptionGroup); +} + void _PyExc_Fini(PyInterpreterState *interp) { struct _Py_exc_state *state = &interp->exc_state; free_preallocated_memerrors(state); Py_CLEAR(state->errnomap); - Py_CLEAR(state->PyExc_ExceptionGroup); } /* Helper to do the equivalent of "raise X from Y" in C, but always using diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c5a209abae61a3..9ce845ca61d213 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1662,6 +1662,8 @@ finalize_interp_clear(PyThreadState *tstate) { int is_main_interp = _Py_IsMainInterpreter(tstate->interp); + _PyExc_ClearExceptionGroupType(tstate->interp); + /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); From ba762bceeacd29f03efe1d61f5192e64f3e1d2c0 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 19 Oct 2021 16:13:04 +0100 Subject: [PATCH 30/40] remove PyExc_ExceptionGroup from stable abi and header --- Doc/data/stable_abi.dat | 1 - Include/pyerrors.h | 1 - Misc/stable_abi.txt | 2 -- PC/python3dll.c | 1 - 4 files changed, 5 deletions(-) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index ff89a1fff8daf5..64a0a2a247cd20 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -204,7 +204,6 @@ var,PyExc_EOFError,3.2, var,PyExc_EncodingWarning,3.10, var,PyExc_EnvironmentError,3.2, var,PyExc_Exception,3.2, -var,PyExc_ExceptionGroup,3.11, var,PyExc_FileExistsError,3.7, var,PyExc_FileNotFoundError,3.7, var,PyExc_FloatingPointError,3.2, diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 87ba2ee28599b8..77d791427d4928 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -68,7 +68,6 @@ PyAPI_FUNC(const char *) PyExceptionClass_Name(PyObject *); PyAPI_DATA(PyObject *) PyExc_BaseException; PyAPI_DATA(PyObject *) PyExc_Exception; PyAPI_DATA(PyObject *) PyExc_BaseExceptionGroup; -PyAPI_DATA(PyObject *) PyExc_ExceptionGroup; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 PyAPI_DATA(PyObject *) PyExc_StopAsyncIteration; #endif diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt index 1383707cf73619..9f5a85bdec40f0 100644 --- a/Misc/stable_abi.txt +++ b/Misc/stable_abi.txt @@ -633,8 +633,6 @@ data PyExc_EnvironmentError added 3.2 data PyExc_Exception added 3.2 -data PyExc_ExceptionGroup - added 3.11 data PyExc_FloatingPointError added 3.2 data PyExc_FutureWarning diff --git a/PC/python3dll.c b/PC/python3dll.c index 41c233f046cbdf..d2a87070de5cce 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -769,7 +769,6 @@ EXPORT_DATA(PyExc_EncodingWarning) EXPORT_DATA(PyExc_EnvironmentError) EXPORT_DATA(PyExc_EOFError) EXPORT_DATA(PyExc_Exception) -EXPORT_DATA(PyExc_ExceptionGroup) EXPORT_DATA(PyExc_FileExistsError) EXPORT_DATA(PyExc_FileNotFoundError) EXPORT_DATA(PyExc_FloatingPointError) From 005cf86fd7658e7abd463888f7f3f38847953bd7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 19 Oct 2021 16:25:06 +0100 Subject: [PATCH 31/40] do type cast only once --- Objects/exceptions.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 78f5126807b05d..fc0866654ef963 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -647,7 +647,8 @@ static PyObject * BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { struct _Py_exc_state *state = get_exc_state(); - PyObject *PyExc_ExceptionGroup = state->PyExc_ExceptionGroup; + PyTypeObject *PyExc_ExceptionGroup = + (PyTypeObject*)state->PyExc_ExceptionGroup; PyObject *message = NULL; PyObject *exceptions = NULL; @@ -701,7 +702,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } PyTypeObject *cls = type; - if (cls == (PyTypeObject*)PyExc_ExceptionGroup) { + if (cls == PyExc_ExceptionGroup) { if (nested_base_exceptions) { PyErr_SetString(PyExc_TypeError, "Cannot nest BaseExceptions in an ExceptionGroup"); @@ -713,7 +714,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* All nested exceptions are Exception subclasses, * wrap them in an ExceptionGroup */ - cls = (PyTypeObject*)PyExc_ExceptionGroup; + cls = PyExc_ExceptionGroup; } } else { @@ -805,7 +806,7 @@ BaseExceptionGroup_derive(PyObject *self_, PyObject *args) static PyObject* exceptiongroup_subset(PyBaseExceptionGroupObject *_orig, PyObject *excs) { - /* Return an ExceptionGroup wrapping excs with metadata from orig. + /* Return an ExceptionGroup wrapping excs with metadata from _orig. This function is used by split() to construct the match/rest parts, so excs is the matching or non-matching sub-sequence of orig->excs From db8c3ea4199e42860dded9a7f88a48fb66f11142 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 20 Oct 2021 11:44:00 +0100 Subject: [PATCH 32/40] add comments and assertions --- Objects/exceptions.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index fc0866654ef963..51072cf2cbfaf7 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -722,7 +722,9 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } if (!cls) { - /* Don't crash during interpreter shutdown */ + /* Don't crash during interpreter shutdown + * (PyExc_ExceptionGroup may have been cleared) + */ cls = (PyTypeObject*)PyExc_BaseExceptionGroup; } PyBaseExceptionGroupObject *self = @@ -854,12 +856,14 @@ typedef enum { EXCEPTION_GROUP_MATCH_BY_TYPE = 0, /* A PyFunction returning True for matching exceptions */ EXCEPTION_GROUP_MATCH_BY_PREDICATE = 1, - /* An instance or container thereof, checked with equality */ + /* An instance or container thereof, checked with equality + * This matcher type is only used internally by the + * interpreter, it is not exposed to python code */ EXCEPTION_GROUP_MATCH_INSTANCES = 2 } _exceptiongroup_split_matcher_type; static int -_get_matcher_type(PyObject *value, +get_matcher_type(PyObject *value, _exceptiongroup_split_matcher_type *type) { /* the python API supports only BY_TYPE and BY_PREDICATE */ @@ -885,9 +889,12 @@ exceptiongroup_split_check_match(PyObject *exc, { switch (matcher_type) { case EXCEPTION_GROUP_MATCH_BY_TYPE: { + assert(PyExceptionClass_Check(matcher_value) || + PyTuple_CheckExact(matcher_value)); return PyErr_GivenExceptionMatches(exc, matcher_value); } case EXCEPTION_GROUP_MATCH_BY_PREDICATE: { + assert(PyFunction_Check(matcher_value)); PyObject *exc_matches = PyObject_CallOneArg(matcher_value, exc); if (exc_matches == NULL) { return -1; @@ -1015,7 +1022,7 @@ BaseExceptionGroup_split(PyObject *self, PyObject *args) } _exceptiongroup_split_matcher_type matcher_type; - if (_get_matcher_type(matcher_value, &matcher_type) == -1) { + if (get_matcher_type(matcher_value, &matcher_type) == -1) { return NULL; } return exceptiongroup_split_recursive( @@ -1031,7 +1038,7 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *args) } _exceptiongroup_split_matcher_type matcher_type; - if (_get_matcher_type(matcher_value, &matcher_type) == -1) { + if (get_matcher_type(matcher_value, &matcher_type) == -1) { return NULL; } PyObject *ret = exceptiongroup_split_recursive( From 7252b3a1431738f84250a4b76ad9aa3dcd3a76c5 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 20 Oct 2021 22:35:51 +0100 Subject: [PATCH 33/40] remove inefficient packing-unpacking of match-rest pairs in a tuple --- Objects/exceptions.c | 151 ++++++++++++++++++++++++++----------------- 1 file changed, 93 insertions(+), 58 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 51072cf2cbfaf7..402462bc1e39ad 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -805,28 +805,33 @@ BaseExceptionGroup_derive(PyObject *self_, PyObject *args) return eg; } -static PyObject* -exceptiongroup_subset(PyBaseExceptionGroupObject *_orig, PyObject *excs) +static int +exceptiongroup_subset( + PyBaseExceptionGroupObject *_orig, PyObject *excs, PyObject **result) { - /* Return an ExceptionGroup wrapping excs with metadata from _orig. + /* Sets *result to an ExceptionGroup wrapping excs with metadata from + * _orig. If excs is empty, sets *result to NULL. + * Returns 0 on success and -1 on error. - This function is used by split() to construct the match/rest parts, - so excs is the matching or non-matching sub-sequence of orig->excs - (this function does not verify that it is a subsequence). - */ + * This function is used by split() to construct the match/rest parts, + * so excs is the matching or non-matching sub-sequence of orig->excs + * (this function does not verify that it is a subsequence). + */ PyObject *orig = (PyObject *)_orig; + + *result = NULL; Py_ssize_t num_excs = PySequence_Size(excs); if (num_excs < 0) { - return NULL; + return -1; } else if (num_excs == 0) { - return Py_NewRef(Py_None); + return 0; } PyObject *eg = PyObject_CallMethod( orig, "derive", "(O)", excs); if (!eg) { - return NULL; + return -1; } if (!_PyBaseExceptionGroup_Check(eg)) { @@ -835,6 +840,8 @@ exceptiongroup_subset(PyBaseExceptionGroupObject *_orig, PyObject *excs) goto error; } + /* Now we hold a reference to the new eg */ + PyObject *tb = PyException_GetTraceback(orig); if (tb) { int res = PyException_SetTraceback(eg, tb); @@ -845,10 +852,11 @@ exceptiongroup_subset(PyBaseExceptionGroupObject *_orig, PyObject *excs) } PyException_SetContext(eg, PyException_GetContext(orig)); PyException_SetCause(eg, PyException_GetCause(orig)); - return eg; + *result = eg; + return 0; error: Py_DECREF(eg); - return NULL; + return -1; } typedef enum { @@ -913,35 +921,47 @@ exceptiongroup_split_check_match(PyObject *exc, return 0; } -static PyObject * +typedef struct { + PyObject *match; + PyObject *rest; +} _exceptiongroup_split_result; + +static int exceptiongroup_split_recursive(PyObject *exc, _exceptiongroup_split_matcher_type matcher_type, PyObject *matcher_value, - int construct_rest) + bool construct_rest, + _exceptiongroup_split_result *result) { + result->match = NULL; + result->rest = NULL; + int is_match = exceptiongroup_split_check_match( exc, matcher_type, matcher_value); if (is_match < 0) { - return NULL; + return -1; } if (is_match) { /* Full match */ - return PyTuple_Pack(2, exc, Py_None); + result->match = Py_NewRef(exc); + return 0; } else if (!_PyBaseExceptionGroup_Check(exc)) { /* Leaf exception and no match */ - return PyTuple_Pack( - 2, Py_None, construct_rest ? (PyObject*)exc : Py_None); + if (construct_rest) { + result->rest = Py_NewRef(exc); + } + return 0; } /* Partial match */ + + int retval = -1; + PyBaseExceptionGroupObject *eg = _PyBaseExceptionGroupObject_cast(exc); PyObject *match_list = NULL; PyObject *rest_list = NULL; - PyObject *match_exc = NULL; - PyObject *rest_exc = NULL; - PyObject *result = NULL; assert(PyTuple_CheckExact(eg->excs)); Py_ssize_t num_excs = PyTuple_Size(eg->excs); @@ -962,55 +982,50 @@ exceptiongroup_split_recursive(PyObject *exc, /* recursive calls */ for (Py_ssize_t i = 0; i < num_excs; i++) { PyObject *e = PyTuple_GET_ITEM(eg->excs, i); - assert(e); - PyObject *rec = exceptiongroup_split_recursive( - e, matcher_type, matcher_value, construct_rest); - if (!rec) { + _exceptiongroup_split_result rec_result; + if (exceptiongroup_split_recursive( + e, matcher_type, matcher_value, + construct_rest, &rec_result) == -1) { + Py_XDECREF(rec_result.match); + Py_XDECREF(rec_result.rest); goto done; } - assert(PyTuple_CheckExact(rec) && PyTuple_GET_SIZE(rec) == 2); - PyObject *e_match = PyTuple_GET_ITEM(rec, 0); - if (e_match != Py_None) { + if (rec_result.match) { assert(PyList_CheckExact(match_list)); - if (PyList_Append(match_list, e_match) < 0) { - Py_DECREF(rec); + if (PyList_Append(match_list, rec_result.match) == -1) { + Py_DECREF(rec_result.match); goto done; } + Py_DECREF(rec_result.match); } - PyObject *e_rest = PyTuple_GET_ITEM(rec, 1); - if (e_rest != Py_None) { + if (rec_result.rest) { assert(construct_rest); assert(PyList_CheckExact(rest_list)); - if (PyList_Append(rest_list, e_rest) < 0) { - Py_DECREF(rec); + if (PyList_Append(rest_list, rec_result.rest) == -1) { + Py_DECREF(rec_result.rest); goto done; } + Py_DECREF(rec_result.rest); } - Py_DECREF(rec); } /* construct result */ - match_exc = exceptiongroup_subset(eg, match_list); - if (!match_exc) { + if (exceptiongroup_subset(eg, match_list, &result->match) == -1) { goto done; } if (construct_rest) { - rest_exc = exceptiongroup_subset(eg, rest_list); - if (!rest_exc) { + assert(PyList_CheckExact(rest_list)); + if (exceptiongroup_subset(eg, rest_list, &result->rest) == -1) { + Py_CLEAR(result->match); goto done; } } - else { - rest_exc = Py_NewRef(Py_None); - } - result = PyTuple_Pack(2, match_exc, rest_exc); + retval = 0; done: - Py_XDECREF(match_exc); - Py_XDECREF(rest_exc); - Py_XDECREF(match_list); - Py_XDECREF(rest_list); - return result; + Py_CLEAR(match_list); + Py_CLEAR(rest_list); + return retval; } static PyObject * @@ -1025,8 +1040,23 @@ BaseExceptionGroup_split(PyObject *self, PyObject *args) if (get_matcher_type(matcher_value, &matcher_type) == -1) { return NULL; } - return exceptiongroup_split_recursive( - self, matcher_type, matcher_value, 1 /* with_construct_rest */); + + PyObject *result = NULL; + _exceptiongroup_split_result split_result; + bool construct_rest = true; + if (exceptiongroup_split_recursive( + self, matcher_type, matcher_value, + construct_rest, &split_result) != -1) { + + result = PyTuple_Pack( + 2, + split_result.match ? split_result.match : Py_None, + split_result.rest ? split_result.rest : Py_None); + } + + Py_XDECREF(split_result.match); + Py_XDECREF(split_result.rest); + return result; } static PyObject * @@ -1041,16 +1071,21 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *args) if (get_matcher_type(matcher_value, &matcher_type) == -1) { return NULL; } - PyObject *ret = exceptiongroup_split_recursive( - self, matcher_type, matcher_value, 0 /* without construct_rest */); - if (!ret) { - return NULL; + PyObject *result = NULL; + _exceptiongroup_split_result split_result; + bool construct_rest = false; + if (exceptiongroup_split_recursive( + self, matcher_type, matcher_value, + construct_rest, &split_result) != -1) { + + result = Py_NewRef( + split_result.match ? split_result.match : Py_None); } - assert(PyTuple_CheckExact(ret) && PyTuple_GET_SIZE(ret) == 2); - PyObject *match = Py_NewRef(PyTuple_GET_ITEM(ret, 0)); - Py_DECREF(ret); - return match; + + Py_XDECREF(split_result.match); + assert(!split_result.rest); + return result; } static PyMemberDef BaseExceptionGroup_members[] = { From ea6878664dc9396faa69ed0a4f7be7ddb748c1a5 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 20 Oct 2021 22:51:57 +0100 Subject: [PATCH 34/40] Py_CLEAR --> Py_DECREF --- Objects/exceptions.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 402462bc1e39ad..e1eadcb942c52f 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1023,8 +1023,8 @@ exceptiongroup_split_recursive(PyObject *exc, } retval = 0; done: - Py_CLEAR(match_list); - Py_CLEAR(rest_list); + Py_DECREF(match_list); + Py_DECREF(rest_list); return retval; } From efa9adb46de93f0d6c23aa28cdbc86e8fda3d96b Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 21 Oct 2021 01:03:44 +0100 Subject: [PATCH 35/40] fix segfault --- Objects/exceptions.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index e1eadcb942c52f..cbd5a86571de54 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1024,7 +1024,7 @@ exceptiongroup_split_recursive(PyObject *exc, retval = 0; done: Py_DECREF(match_list); - Py_DECREF(rest_list); + Py_XDECREF(rest_list); return retval; } From cd61b77dc3839ab123ee88a4b113dd09fe7d38d7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 21 Oct 2021 10:34:12 +0100 Subject: [PATCH 36/40] clear match and rest in exceptiongroup_split_recursive when there is an error. Tidy up error handling. --- Objects/exceptions.c | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index cbd5a86571de54..a7148f788338b7 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -957,22 +957,21 @@ exceptiongroup_split_recursive(PyObject *exc, /* Partial match */ - int retval = -1; - PyBaseExceptionGroupObject *eg = _PyBaseExceptionGroupObject_cast(exc); - PyObject *match_list = NULL; - PyObject *rest_list = NULL; - assert(PyTuple_CheckExact(eg->excs)); Py_ssize_t num_excs = PyTuple_Size(eg->excs); if (num_excs < 0) { - goto done; + return -1; } assert(num_excs > 0); /* checked in constructor, and excs is read-only */ - match_list = PyList_New(0); + + int retval = -1; + PyObject *match_list = PyList_New(0); if (!match_list) { - goto done; + return -1; } + + PyObject *rest_list = NULL; if (construct_rest) { rest_list = PyList_New(0); if (!rest_list) { @@ -986,8 +985,8 @@ exceptiongroup_split_recursive(PyObject *exc, if (exceptiongroup_split_recursive( e, matcher_type, matcher_value, construct_rest, &rec_result) == -1) { - Py_XDECREF(rec_result.match); - Py_XDECREF(rec_result.rest); + assert(!rec_result.match); + assert(!rec_result.rest); goto done; } if (rec_result.match) { @@ -1025,6 +1024,10 @@ exceptiongroup_split_recursive(PyObject *exc, done: Py_DECREF(match_list); Py_XDECREF(rest_list); + if (retval == -1) { + Py_CLEAR(result->match); + Py_CLEAR(result->rest); + } return retval; } @@ -1041,18 +1044,18 @@ BaseExceptionGroup_split(PyObject *self, PyObject *args) return NULL; } - PyObject *result = NULL; _exceptiongroup_split_result split_result; bool construct_rest = true; if (exceptiongroup_split_recursive( self, matcher_type, matcher_value, - construct_rest, &split_result) != -1) { + construct_rest, &split_result) == -1) { + return NULL; + } - result = PyTuple_Pack( + PyObject *result = PyTuple_Pack( 2, split_result.match ? split_result.match : Py_None, split_result.rest ? split_result.rest : Py_None); - } Py_XDECREF(split_result.match); Py_XDECREF(split_result.rest); @@ -1072,16 +1075,16 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *args) return NULL; } - PyObject *result = NULL; _exceptiongroup_split_result split_result; bool construct_rest = false; if (exceptiongroup_split_recursive( self, matcher_type, matcher_value, - construct_rest, &split_result) != -1) { + construct_rest, &split_result) == -1) { + return NULL; + } - result = Py_NewRef( + PyObject *result = Py_NewRef( split_result.match ? split_result.match : Py_None); - } Py_XDECREF(split_result.match); assert(!split_result.rest); From c463d4cc63039aebc7f323f927ccca56d33d4870 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 21 Oct 2021 22:29:37 +0100 Subject: [PATCH 37/40] make the tests less crazy --- Lib/test/test_exception_group.py | 459 ++++++++++++++++--------------- 1 file changed, 240 insertions(+), 219 deletions(-) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index f544d7ef309afa..432ea690bc0655 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -49,7 +49,8 @@ def test_bad_EG_construction__bad_excs_sequence(self): ExceptionGroup("eg", []) def test_bad_EG_construction__nested_non_exceptions(self): - MSG = 'Item [0-9]+ of second argument \(exceptions\) is not an exception' + MSG = ('Item [0-9]+ of second argument \(exceptions\)' + ' is not an exception') with self.assertRaisesRegex(ValueError, MSG): ExceptionGroup('expect instance, not type', [OSError]); with self.assertRaisesRegex(ValueError, MSG): @@ -59,11 +60,15 @@ def test_bad_EG_construction__nested_non_exceptions(self): class InstanceCreation(unittest.TestCase): def test_EG_wraps_Exceptions__creates_EG(self): excs = [ValueError(1), TypeError(2)] - self.assertIs(type(ExceptionGroup("eg", excs)), ExceptionGroup) + self.assertIs( + type(ExceptionGroup("eg", excs)), + ExceptionGroup) def test_BEG_wraps_Exceptions__creates_EG(self): excs = [ValueError(1), TypeError(2)] - self.assertIs(type(BaseExceptionGroup("beg", excs)), ExceptionGroup) + self.assertIs( + type(BaseExceptionGroup("beg", excs)), + ExceptionGroup) def test_EG_wraps_BaseException__raises_TypeError(self): MSG= "Cannot nest BaseExceptions in an ExceptionGroup" @@ -77,53 +82,100 @@ def test_BEG_wraps_BaseException__creates_BEG(self): def test_EG_subclass_wraps_anything(self): class MyEG(ExceptionGroup): pass + self.assertIs( - type(MyEG("eg", [ValueError(12), TypeError(42)])), MyEG) + type(MyEG("eg", [ValueError(12), TypeError(42)])), + MyEG) self.assertIs( - type(MyEG("eg", [ValueError(12), KeyboardInterrupt(42)])), MyEG) + type(MyEG("eg", [ValueError(12), KeyboardInterrupt(42)])), + MyEG) def test_BEG_subclass_wraps_anything(self): class MyBEG(BaseExceptionGroup): pass + self.assertIs( - type(MyBEG("eg", [ValueError(12), TypeError(42)])), MyBEG) + type(MyBEG("eg", [ValueError(12), TypeError(42)])), + MyBEG) self.assertIs( - type(MyBEG("eg", [ValueError(12), KeyboardInterrupt(42)])), MyBEG) + type(MyBEG("eg", [ValueError(12), KeyboardInterrupt(42)])), + MyBEG) -def leaf_generator(exc, tbs=None): - if tbs is None: - tbs = [] - tbs.append(exc.__traceback__) - if isinstance(exc, BaseExceptionGroup): - for e in exc.exceptions: - yield from leaf_generator(e, tbs) - else: - # exc is a leaf exception and its traceback - # is the concatenation of the traceback - # segments in tbs - yield exc, tbs - tbs.pop() +def create_simple_eg(): + excs = [] + try: + try: + raise MemoryError("context and cause for ValueError(1)") + except MemoryError as e: + raise ValueError(1) from e + except ValueError as e: + excs.append(e) + try: + try: + raise OSError("context for TypeError") + except OSError as e: + raise TypeError(int) + except TypeError as e: + excs.append(e) -def tb_funcnames(tbs): - # tbs is a traceback or a list of tracebacks - def names_list(tb): - if tb is None: - return None - else: - return [f.name for f in traceback.extract_tb(tb)] + try: + try: + raise ImportError("context for ValueError(2)") + except ImportError as e: + raise ValueError(2) + except ValueError as e: + excs.append(e) - if isinstance(tbs, types.TracebackType): - return names_list(tbs) - else: # list of tracebacks - return [names_list(tb) for tb in tbs] + try: + raise ExceptionGroup('simple eg', excs) + except ExceptionGroup as e: + return e -def tbs_for_leaf(leaf, eg): - for e, tbs in leaf_generator(eg): - if e is leaf: - return tbs +class ExceptionGroupFields(unittest.TestCase): + def test_basics_ExceptionGroup_fields(self): + eg = create_simple_eg() + + # check msg + self.assertEqual(eg.message, 'simple eg') + self.assertEqual(eg.args[0], 'simple eg') + + # check cause and context + self.assertIsInstance(eg.exceptions[0], ValueError) + self.assertIsInstance(eg.exceptions[0].__cause__, MemoryError) + self.assertIsInstance(eg.exceptions[0].__context__, MemoryError) + self.assertIsInstance(eg.exceptions[1], TypeError) + self.assertIsNone(eg.exceptions[1].__cause__) + self.assertIsInstance(eg.exceptions[1].__context__, OSError) + self.assertIsInstance(eg.exceptions[2], ValueError) + self.assertIsNone(eg.exceptions[2].__cause__) + self.assertIsInstance(eg.exceptions[2].__context__, ImportError) + + # check tracebacks + line0 = create_simple_eg.__code__.co_firstlineno + tb_linenos = [line0 + 27, + [line0 + 6, line0 + 14, line0 + 22]] + self.assertEqual(eg.__traceback__.tb_lineno, tb_linenos[0]) + self.assertIsNone(eg.__traceback__.tb_next) + for i in range(3): + tb = eg.exceptions[i].__traceback__ + self.assertIsNone(tb.tb_next) + self.assertEqual(tb.tb_lineno, tb_linenos[1][i]) + + def test_fields_are_readonly(self): + eg = ExceptionGroup('eg', [TypeError(1), OSError(2)]) + + self.assertEqual(type(eg.exceptions), tuple) + + eg.message + with self.assertRaises(AttributeError): + eg.message = "new msg" + + eg.exceptions + with self.assertRaises(AttributeError): + eg.exceptions = [OSError('xyz')] class ExceptionGroupTestBase(unittest.TestCase): @@ -151,115 +203,11 @@ def assertMatchesTemplate(self, exc, exc_type, template): self.assertEqual(exc.args, template.args) -class ExceptionGroupBasicsTests(ExceptionGroupTestBase): +class ExceptionGroupSubgroupTests(ExceptionGroupTestBase): def setUp(self): - # simple ExceptionGroup (without nesting) - - def do_raise(type, arg, cause=None): - if cause is None: - raise type(arg) - else: - raise type(arg) from cause - - def getTypeError(v): - try: - try: - do_raise(OSError, "context for TypeError") - except: - do_raise(TypeError, v) - except Exception as e: - return e - - def getValueError(v): - try: - try: - do_raise(MemoryError, "context and cause for ValueError") - except MemoryError as e: - do_raise(ValueError, v, cause=e) - except Exception as e: - return e - - def simple_group(): - try: - raise ExceptionGroup( - 'simple EG', - [getValueError(1), getTypeError(int), getValueError(2)]) - except ExceptionGroup as e: - return e - - try: - raise simple_group() - except Exception as e: - self.eg = e - + self.eg = create_simple_eg() self.eg_template = [ValueError(1), TypeError(int), ValueError(2)] - self.root_tb_fnames = ['setUp', 'simple_group'] - self.value_error_tb_fnames = ['getValueError', 'do_raise'] - self.type_error_tb_fnames = ['getTypeError', 'do_raise'] - - def test_basics_ExceptionGroup_fields(self): - eg = self.eg - - self.assertMatchesTemplate(eg, ExceptionGroup, self.eg_template) - - # check msg - self.assertEqual(eg.message, 'simple EG') - self.assertEqual(eg.args[0], 'simple EG') - - # check cause and context - self.assertIsInstance(eg.exceptions[0], ValueError) - self.assertIsInstance(eg.exceptions[0].__cause__, MemoryError) - self.assertIsInstance(eg.exceptions[0].__context__, MemoryError) - self.assertIsInstance(eg.exceptions[1], TypeError) - self.assertIsNone(eg.exceptions[1].__cause__) - self.assertIsInstance(eg.exceptions[1].__context__, OSError) - self.assertIsInstance(eg.exceptions[2], ValueError) - self.assertIsInstance(eg.exceptions[2].__cause__, MemoryError) - self.assertIsInstance(eg.exceptions[2].__context__, MemoryError) - - # check tracebacks - self.assertSequenceEqual( - tb_funcnames(eg.__traceback__), self.root_tb_fnames) - - self.assertSequenceEqual( - tb_funcnames(eg.exceptions[0].__traceback__), - self.value_error_tb_fnames) - self.assertSequenceEqual( - tb_funcnames(eg.exceptions[1].__traceback__), - self.type_error_tb_fnames) - self.assertSequenceEqual( - tb_funcnames(eg.exceptions[2].__traceback__), - self.value_error_tb_fnames) - - - def test_basics_ExceptionGroup_fields_are_read_only(self): - eg = self.eg - self.assertEqual(type(eg.exceptions), tuple) - - eg.message - with self.assertRaises(AttributeError): - eg.message = "new msg" - eg.exceptions - with self.assertRaises(AttributeError): - eg.exceptions = [OSError('xyz')] - - def test_basics_leaf_generator(self): - eg = self.eg - self.assertSequenceEqual( - [e for e, _ in leaf_generator(eg)], eg.exceptions) - - self.assertSequenceEqual( - [tb_funcnames(tbs) for _, tbs in leaf_generator(eg)], - [[self.root_tb_fnames, self.value_error_tb_fnames], - [self.root_tb_fnames, self.type_error_tb_fnames], - [self.root_tb_fnames, self.value_error_tb_fnames]]) - - def test_basics__tbs_for_leaf__helper_works(self): - eg = self.eg - for e, tbs in leaf_generator(eg): - self.assertSequenceEqual(tbs, tbs_for_leaf(e, eg)) - def test_basics_subgroup_split__bad_arg_type(self): bad_args = ["bad arg", OSError('instance not type'), @@ -289,9 +237,10 @@ def test_basics_subgroup_by_type__match(self): ((ValueError, TypeError), self.eg_template)] for match_type, template in testcases: - subeg = eg.subgroup(match_type) - self.assertEqual(subeg.message, eg.message) - self.assertMatchesTemplate(subeg, ExceptionGroup, template) + with self.subTest(match=match_type): + subeg = eg.subgroup(match_type) + self.assertEqual(subeg.message, eg.message) + self.assertMatchesTemplate(subeg, ExceptionGroup, template) def test_basics_subgroup_by_predicate__passthrough(self): self.assertIs(self.eg, self.eg.subgroup(lambda e: True)) @@ -312,17 +261,25 @@ def test_basics_subgroup_by_predicate__match(self): self.assertEqual(subeg.message, eg.message) self.assertMatchesTemplate(subeg, ExceptionGroup, template) + +class ExceptionGroupSplitTests(ExceptionGroupTestBase): + def setUp(self): + self.eg = create_simple_eg() + self.eg_template = [ValueError(1), TypeError(int), ValueError(2)] + def test_basics_split_by_type__passthrough(self): for E in [BaseException, Exception, BaseExceptionGroup, ExceptionGroup]: match, rest = self.eg.split(E) - self.assertMatchesTemplate(match, ExceptionGroup, self.eg_template) + self.assertMatchesTemplate( + match, ExceptionGroup, self.eg_template) self.assertIsNone(rest) def test_basics_split_by_type__no_match(self): match, rest = self.eg.split(OSError) self.assertIsNone(match) - self.assertMatchesTemplate(rest, ExceptionGroup, self.eg_template) + self.assertMatchesTemplate( + rest, ExceptionGroup, self.eg_template) def test_basics_split_by_type__match(self): eg = self.eg @@ -380,67 +337,108 @@ def test_basics_split_by_predicate__match(self): rest, ExceptionGroup, rest_template) -class NestedExceptionGroupBasicsTest(ExceptionGroupTestBase): - def setUp(self): - def raiseVE(v): - raise ValueError(v) +def leaf_generator(exc, tbs=None): + if tbs is None: + tbs = [] + tbs.append(exc.__traceback__) + if isinstance(exc, BaseExceptionGroup): + for e in exc.exceptions: + yield from leaf_generator(e, tbs) + else: + # exc is a leaf exception and its traceback + # is the concatenation of the traceback + # segments in tbs + yield exc, tbs + tbs.pop() - def raiseTE(t): - raise TypeError(t) - def nested_group(): - try: - try: - raiseTE(bytes) - except TypeError as e: - raise ExceptionGroup("nested", [e]) - except ExceptionGroup as e: - excs = [e] - try: - raiseVE(1) - except ValueError as e: - excs.append(e) - raise ExceptionGroup("root", excs) +class LeafGeneratorTest(unittest.TestCase): + # The leaf_generator is mentioned in PEP 654 as a suggestion + # on how to iterate over leaf nodes of an EG. Is is also + # used below as a test utility. So we test it here. + + def test_leaf_generator(self): + eg = create_simple_eg() + + self.assertSequenceEqual( + [e for e, _ in leaf_generator(eg)], + eg.exceptions) + for e, tbs in leaf_generator(eg): + self.assertSequenceEqual( + tbs, [eg.__traceback__, e.__traceback__]) + + +def create_nested_eg(): + excs = [] + try: try: - nested_group() - except ExceptionGroup as eg: - self.eg = eg + raise TypeError(bytes) + except TypeError as e: + raise ExceptionGroup("nested", [e]) + except ExceptionGroup as e: + excs.append(e) - def test_nested_group_matches_template(self): - self.assertMatchesTemplate( - self.eg, ExceptionGroup, [[TypeError(bytes)], ValueError(1)]) + try: + try: + raise MemoryError('out of memory') + except MemoryError as e: + raise ValueError(1) from e + except ValueError as e: + excs.append(e) + + try: + raise ExceptionGroup("root", excs) + except ExceptionGroup as eg: + return eg - def test_nested_group_tracebacks(self): - eg = self.eg - def check(exc, expected): - self.assertEqual(tb_funcnames(exc.__traceback__), expected) - check(eg, ['setUp', 'nested_group']) - check(eg.exceptions[0], ['nested_group']) - check(eg.exceptions[1], ['nested_group', 'raiseVE']) - check(eg.exceptions[0].exceptions[0], ['nested_group', 'raiseTE']) +class NestedExceptionGroupBasicsTest(ExceptionGroupTestBase): + def test_nested_group_matches_template(self): + eg = create_nested_eg() + self.assertMatchesTemplate( + eg, + ExceptionGroup, + [[TypeError(bytes)], ValueError(1)]) + + def test_nested_group_chaining(self): + eg = create_nested_eg() + self.assertIsInstance(eg.exceptions[1].__context__, MemoryError) + self.assertIsInstance(eg.exceptions[1].__cause__, MemoryError) + self.assertIsInstance(eg.exceptions[0].__context__, TypeError) + + def test_nested_exception_group_tracebacks(self): + eg = create_nested_eg() + + line0 = create_nested_eg.__code__.co_firstlineno + for (tb, expected) in [ + (eg.__traceback__, line0 + 19), + (eg.exceptions[0].__traceback__, line0 + 6), + (eg.exceptions[1].__traceback__, line0 + 14), + (eg.exceptions[0].exceptions[0].__traceback__, line0 + 4), + ]: + self.assertEqual(tb.tb_lineno, expected) + self.assertIsNone(tb.tb_next) def test_iteration_full_tracebacks(self): - eg = self.eg + eg = create_nested_eg() # check that iteration over leaves # produces the expected tracebacks self.assertEqual(len(list(leaf_generator(eg))), 2) - expected_tbs = [ - [['setUp', 'nested_group'], - ['nested_group'], - ['nested_group', 'raiseTE']], - [['setUp', 'nested_group'], - ['nested_group', 'raiseVE']]] + line0 = create_nested_eg.__code__.co_firstlineno + expected_tbs = [ [line0 + 19, line0 + 6, line0 + 4], + [line0 + 19, line0 + 14]] - for (i, (e, tbs)) in enumerate(leaf_generator(eg)): + for (i, (_, tbs)) in enumerate(leaf_generator(eg)): self.assertSequenceEqual( - expected_tbs[i], [tb_funcnames(tb) for tb in tbs]) + [tb.tb_lineno for tb in tbs], + expected_tbs[i]) -class ExceptionGroupSplitTests(ExceptionGroupTestBase): - def _split_exception_group(self, eg, types): +class ExceptionGroupSplitTestBase(ExceptionGroupTestBase): + + def split_exception_group(self, eg, types): """ Split an EG and do some sanity checks on the result """ self.assertIsInstance(eg, BaseExceptionGroup) @@ -468,8 +466,11 @@ def leaves(exc): match_leaves = leaves(match) rest_leaves = leaves(rest) - # each leaf exception of eg and exactly one of match and rest - self.assertEqual(len(leaves(eg)), len(leaves(match)) + len(leaves(rest))) + # each leaf exception of eg is in exactly one of match and rest + self.assertEqual( + len(leaves(eg)), + len(leaves(match)) + len(leaves(rest))) + for e in leaves(eg): self.assertNotEqual( match and e in match_leaves, @@ -483,15 +484,26 @@ def leaves(exc): self.assertIs(eg.__context__, part.__context__) self.assertIs(eg.__traceback__, part.__traceback__) + def tbs_for_leaf(leaf, eg): + for e, tbs in leaf_generator(eg): + if e is leaf: + return tbs + + def tb_linenos(tbs): + return [tb.tb_lineno for tb in tbs if tb] + # full tracebacks match for part in [match, rest, sg]: for e in leaves(part): self.assertSequenceEqual( - tb_funcnames(tbs_for_leaf(e, eg)), - tb_funcnames(tbs_for_leaf(e, part))) + tb_linenos(tbs_for_leaf(e, eg)), + tb_linenos(tbs_for_leaf(e, part))) return match, rest + +class NestedExceptionGroupSplitTest(ExceptionGroupSplitTestBase): + def test_split_by_type(self): class MyExceptionGroup(ExceptionGroup): pass @@ -558,25 +570,25 @@ def level3(i): self.assertMatchesTemplate(eg, ExceptionGroup, eg_template) # Match Nothing - match, rest = self._split_exception_group(eg, SyntaxError) + match, rest = self.split_exception_group(eg, SyntaxError) self.assertIsNone(match) self.assertMatchesTemplate(rest, ExceptionGroup, eg_template) # Match Everything - match, rest = self._split_exception_group(eg, BaseException) + match, rest = self.split_exception_group(eg, BaseException) self.assertMatchesTemplate(match, ExceptionGroup, eg_template) self.assertIsNone(rest) - match, rest = self._split_exception_group(eg, (ValueError, TypeError)) + match, rest = self.split_exception_group(eg, (ValueError, TypeError)) self.assertMatchesTemplate(match, ExceptionGroup, eg_template) self.assertIsNone(rest) # Match ValueErrors - match, rest = self._split_exception_group(eg, ValueError) + match, rest = self.split_exception_group(eg, ValueError) self.assertMatchesTemplate(match, ExceptionGroup, valueErrors_template) self.assertMatchesTemplate(rest, ExceptionGroup, typeErrors_template) # Match TypeErrors - match, rest = self._split_exception_group(eg, (TypeError, SyntaxError)) + match, rest = self.split_exception_group(eg, (TypeError, SyntaxError)) self.assertMatchesTemplate(match, ExceptionGroup, typeErrors_template) self.assertMatchesTemplate(rest, ExceptionGroup, valueErrors_template) @@ -604,26 +616,34 @@ def exc(ex): beg = e # Match Nothing - match, rest = self._split_exception_group(beg, TypeError) + match, rest = self.split_exception_group(beg, TypeError) self.assertIsNone(match) self.assertMatchesTemplate( rest, BaseExceptionGroup, [ValueError(1), KeyboardInterrupt(2)]) # Match Everything - match, rest = self._split_exception_group(beg, (ValueError, KeyboardInterrupt)) + match, rest = self.split_exception_group( + beg, (ValueError, KeyboardInterrupt)) self.assertMatchesTemplate( match, BaseExceptionGroup, [ValueError(1), KeyboardInterrupt(2)]) self.assertIsNone(rest) # Match ValueErrors - match, rest = self._split_exception_group(beg, ValueError) - self.assertMatchesTemplate(match, ExceptionGroup, [ValueError(1)]) - self.assertMatchesTemplate(rest, BaseExceptionGroup, [KeyboardInterrupt(2)]) + match, rest = self.split_exception_group(beg, ValueError) + self.assertMatchesTemplate( + match, ExceptionGroup, [ValueError(1)]) + self.assertMatchesTemplate( + rest, BaseExceptionGroup, [KeyboardInterrupt(2)]) # Match KeyboardInterrupts - match, rest = self._split_exception_group(beg, KeyboardInterrupt) - self.assertMatchesTemplate(match, BaseExceptionGroup, [KeyboardInterrupt(2)]) - self.assertMatchesTemplate(rest, ExceptionGroup, [ValueError(1)]) + match, rest = self.split_exception_group(beg, KeyboardInterrupt) + self.assertMatchesTemplate( + match, BaseExceptionGroup, [KeyboardInterrupt(2)]) + self.assertMatchesTemplate( + rest, ExceptionGroup, [ValueError(1)]) + + +class NestedExceptionGroupSubclassSplitTest(ExceptionGroupSplitTestBase): def test_split_ExceptionGroup_subclass_no_derive_no_new_override(self): class EG(ExceptionGroup): @@ -646,24 +666,24 @@ class EG(ExceptionGroup): self.assertMatchesTemplate(eg, EG, [ValueError(1), [TypeError(2)]]) # Match Nothing - match, rest = self._split_exception_group(eg, OSError) + match, rest = self.split_exception_group(eg, OSError) self.assertIsNone(match) self.assertMatchesTemplate( rest, ExceptionGroup, [ValueError(1), [TypeError(2)]]) # Match Everything - match, rest = self._split_exception_group(eg, (ValueError, TypeError)) + match, rest = self.split_exception_group(eg, (ValueError, TypeError)) self.assertMatchesTemplate( match, ExceptionGroup, [ValueError(1), [TypeError(2)]]) self.assertIsNone(rest) # Match ValueErrors - match, rest = self._split_exception_group(eg, ValueError) + match, rest = self.split_exception_group(eg, ValueError) self.assertMatchesTemplate(match, ExceptionGroup, [ValueError(1)]) self.assertMatchesTemplate(rest, ExceptionGroup, [[TypeError(2)]]) # Match TypeErrors - match, rest = self._split_exception_group(eg, TypeError) + match, rest = self.split_exception_group(eg, TypeError) self.assertMatchesTemplate(match, ExceptionGroup, [[TypeError(2)]]) self.assertMatchesTemplate(rest, ExceptionGroup, [ValueError(1)]) @@ -685,25 +705,26 @@ def __new__(cls, message, excs, unused): eg, EG, [ValueError(1), KeyboardInterrupt(2)]) # Match Nothing - match, rest = self._split_exception_group(eg, OSError) + match, rest = self.split_exception_group(eg, OSError) self.assertIsNone(match) self.assertMatchesTemplate( rest, BaseExceptionGroup, [ValueError(1), KeyboardInterrupt(2)]) # Match Everything - match, rest = self._split_exception_group(eg, (ValueError, KeyboardInterrupt)) + match, rest = self.split_exception_group( + eg, (ValueError, KeyboardInterrupt)) self.assertMatchesTemplate( match, BaseExceptionGroup, [ValueError(1), KeyboardInterrupt(2)]) self.assertIsNone(rest) # Match ValueErrors - match, rest = self._split_exception_group(eg, ValueError) + match, rest = self.split_exception_group(eg, ValueError) self.assertMatchesTemplate(match, ExceptionGroup, [ValueError(1)]) self.assertMatchesTemplate( rest, BaseExceptionGroup, [KeyboardInterrupt(2)]) # Match KeyboardInterrupt - match, rest = self._split_exception_group(eg, KeyboardInterrupt) + match, rest = self.split_exception_group(eg, KeyboardInterrupt) self.assertMatchesTemplate( match, BaseExceptionGroup, [KeyboardInterrupt(2)]) self.assertMatchesTemplate(rest, ExceptionGroup, [ValueError(1)]) @@ -735,21 +756,21 @@ def derive(self, excs): self.assertMatchesTemplate(eg, EG, [ValueError(1), [TypeError(2)]]) # Match Nothing - match, rest = self._split_exception_group(eg, OSError) + match, rest = self.split_exception_group(eg, OSError) self.assertIsNone(match) self.assertMatchesTemplate(rest, EG, [ValueError(1), [TypeError(2)]]) self.assertEqual(rest.code, 42) self.assertEqual(rest.exceptions[1].code, 101) # Match Everything - match, rest = self._split_exception_group(eg, (ValueError, TypeError)) + match, rest = self.split_exception_group(eg, (ValueError, TypeError)) self.assertMatchesTemplate(match, EG, [ValueError(1), [TypeError(2)]]) self.assertEqual(match.code, 42) self.assertEqual(match.exceptions[1].code, 101) self.assertIsNone(rest) # Match ValueErrors - match, rest = self._split_exception_group(eg, ValueError) + match, rest = self.split_exception_group(eg, ValueError) self.assertMatchesTemplate(match, EG, [ValueError(1)]) self.assertEqual(match.code, 42) self.assertMatchesTemplate(rest, EG, [[TypeError(2)]]) @@ -757,7 +778,7 @@ def derive(self, excs): self.assertEqual(rest.exceptions[0].code, 101) # Match TypeErrors - match, rest = self._split_exception_group(eg, TypeError) + match, rest = self.split_exception_group(eg, TypeError) self.assertMatchesTemplate(match, EG, [[TypeError(2)]]) self.assertEqual(match.code, 42) self.assertEqual(match.exceptions[0].code, 101) From 4d76ccc1c41f8b96b799c3533d60a1deee7a1b4d Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 22 Oct 2021 13:39:43 +0100 Subject: [PATCH 38/40] make regen-limited-abi --- Lib/test/test_stable_abi_ctypes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 750aa18108327e..1e27bcaf889a21 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -198,6 +198,7 @@ def test_available_symbols(self): "PyExc_AssertionError", "PyExc_AttributeError", "PyExc_BaseException", + "PyExc_BaseExceptionGroup", "PyExc_BlockingIOError", "PyExc_BrokenPipeError", "PyExc_BufferError", From 26b042666aed9a78d2191a0778ef45efa1407524 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Fri, 22 Oct 2021 12:06:59 -0700 Subject: [PATCH 39/40] use :pep: syntax in the news entry. --- .../Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst index 0fe7720094381a..ee48b6d5105c52 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst @@ -1 +1 @@ -Implement PEP 654: Add :class:`ExceptionGroup` and :class:`BaseExceptionGroup`. \ No newline at end of file +Implement :pep:`654` Add :class:`ExceptionGroup` and :class:`BaseExceptionGroup`. From dbd72d12909bdf8e31f3db5b25a93445f11df2bd Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 22 Oct 2021 21:44:10 +0100 Subject: [PATCH 40/40] add recursion guard in split --- Lib/test/test_exception_group.py | 18 ++++++++++++++++++ Objects/exceptions.c | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 432ea690bc0655..5bb6094cde742c 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -337,6 +337,24 @@ def test_basics_split_by_predicate__match(self): rest, ExceptionGroup, rest_template) +class DeepRecursionInSplitAndSubgroup(unittest.TestCase): + def make_deep_eg(self): + e = TypeError(1) + for i in range(2000): + e = ExceptionGroup('eg', [e]) + return e + + def test_deep_split(self): + e = self.make_deep_eg() + with self.assertRaises(RecursionError): + e.split(TypeError) + + def test_deep_subgroup(self): + e = self.make_deep_eg() + with self.assertRaises(RecursionError): + e.subgroup(TypeError) + + def leaf_generator(exc, tbs=None): if tbs is None: tbs = [] diff --git a/Objects/exceptions.c b/Objects/exceptions.c index a7148f788338b7..a5459da89a073d 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -982,13 +982,18 @@ exceptiongroup_split_recursive(PyObject *exc, for (Py_ssize_t i = 0; i < num_excs; i++) { PyObject *e = PyTuple_GET_ITEM(eg->excs, i); _exceptiongroup_split_result rec_result; + if (Py_EnterRecursiveCall(" in exceptiongroup_split_recursive")) { + goto done; + } if (exceptiongroup_split_recursive( e, matcher_type, matcher_value, construct_rest, &rec_result) == -1) { assert(!rec_result.match); assert(!rec_result.rest); + Py_LeaveRecursiveCall(); goto done; } + Py_LeaveRecursiveCall(); if (rec_result.match) { assert(PyList_CheckExact(match_list)); if (PyList_Append(match_list, rec_result.match) == -1) {