From f2a8b9016cab3633b3c0db9d52e2d82b19d9b622 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 2 Sep 2024 20:36:59 +0300 Subject: [PATCH 1/2] gh-123339: Fix cases of inconsistency of __module__ and __firstlineno__ in classes * Setting the __module__ attribute for a class now removes the __firstlineno__ item from the type's dict. * The _collections_abc and _pydecimal modules now completely replace the collections.abc and decimal modules after importing the latters. This allows to get the source of classes and functions defined in these modules. * inspect.findsource() now checks whether the first line number for a class is out of bound. --- Doc/reference/datamodel.rst | 1 + Lib/collections/abc.py | 6 +- Lib/decimal.py | 7 ++- Lib/inspect.py | 6 +- Lib/test/test_builtin.py | 12 ++++ Lib/test/test_decimal.py | 3 +- Lib/test/test_inspect/inspect_fodder2.py | 12 ++++ Lib/test/test_inspect/test_inspect.py | 61 ++++++++++++++++++- ...-09-02-20-36-45.gh-issue-123339.QcmpSs.rst | 3 + ...-09-02-20-34-04.gh-issue-123339.czgcSu.rst | 4 ++ Objects/typeobject.c | 3 + 11 files changed, 107 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst create mode 100644 Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 8c3134a61ed1a6..2cfadee179ebfb 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1008,6 +1008,7 @@ Special attributes: :attr:`__firstlineno__` The line number of the first line of the class definition, including decorators. + Setting :attr:`__module__` removes :attr:`__firstlineno__`. Class instances diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index bff76291634604..034ba377a0dbec 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,3 @@ -from _collections_abc import * -from _collections_abc import __all__ # noqa: F401 -from _collections_abc import _CallableGenericAlias # noqa: F401 +import _collections_abc +import sys +sys.modules[__name__] = _collections_abc diff --git a/Lib/decimal.py b/Lib/decimal.py index f8c548eb1c6ecf..530bdfb38953d9 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -103,6 +103,7 @@ from _decimal import __version__ # noqa: F401 from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: - from _pydecimal import * - from _pydecimal import __version__ # noqa: F401 - from _pydecimal import __libmpdec_version__ # noqa: F401 + import _pydecimal + import sys + _pydecimal.__doc__ = __doc__ + sys.modules[__name__] = _pydecimal diff --git a/Lib/inspect.py b/Lib/inspect.py index 90c44cf74007a8..a4f03965d13662 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -970,10 +970,12 @@ def findsource(object): if isclass(object): try: - firstlineno = vars(object)['__firstlineno__'] + lnum = vars(object)['__firstlineno__'] - 1 except (TypeError, KeyError): raise OSError('source code not available') - return lines, firstlineno - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') + return lines, lnum if ismethod(object): object = object.__func__ diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 2ea97e797a4892..d884f54940b471 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2607,6 +2607,7 @@ def test_new_type(self): self.assertEqual(A.__module__, __name__) self.assertEqual(A.__bases__, (object,)) self.assertIs(A.__base__, object) + self.assertNotIn('__firstlineno__', A.__dict__) x = A() self.assertIs(type(x), A) self.assertIs(x.__class__, A) @@ -2685,6 +2686,17 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') + def test_type_firstlineno(self): + A = type('A', (), {'__firstlineno__': 42}) + self.assertEqual(A.__name__, 'A') + self.assertEqual(A.__module__, __name__) + self.assertEqual(A.__dict__['__firstlineno__'], 42) + A.__module__ = 'testmodule' + self.assertEqual(A.__module__, 'testmodule') + self.assertNotIn('__firstlineno__', A.__dict__) + A.__firstlineno__ = 43 + self.assertEqual(A.__dict__['__firstlineno__'], 43) + def test_type_typeparams(self): class A[T]: pass diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 46755107de0102..9fb7775d5d5e86 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4379,7 +4379,8 @@ def test_module_attributes(self): self.assertEqual(C.__version__, P.__version__) - self.assertEqual(dir(C), dir(P)) + self.assertLessEqual(set(dir(C)), set(dir(P))) + self.assertEqual([n for n in dir(C) if n[:2] != '__'], sorted(P.__all__)) def test_context_attributes(self): diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index 43e9f852022934..43fda6622537fc 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -357,3 +357,15 @@ class td354(typing.TypedDict): # line 358 td359 = typing.TypedDict('td359', (('x', int), ('y', int))) + +import dataclasses + +# line 363 +@dataclasses.dataclass +class dc364: + x: int + y: int + +# line 369 +dc370 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int))) +dc371 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int)), module=__name__) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 81188ad4d1fbe1..6174997cba7b36 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -835,6 +835,47 @@ class C: nonlocal __firstlineno__ self.assertRaises(OSError, inspect.getsource, C) +class TestGetsourceStdlib(unittest.TestCase): + # Test Python implementations of the stdlib modules + + def test_getsource_stdlib_collections_abc(self): + import collections.abc + lines, lineno = inspect.getsourcelines(collections.abc.Sequence) + self.assertEqual(lines[0], 'class Sequence(Reversible, Collection):\n') + src = inspect.getsource(collections.abc.Sequence) + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_tomllib(self): + import tomllib + self.assertRaises(OSError, inspect.getsource, tomllib.TOMLDecodeError) + self.assertRaises(OSError, inspect.getsourcelines, tomllib.TOMLDecodeError) + + def test_getsource_stdlib_abc(self): + # Pure Python implementation + abc = import_helper.import_fresh_module('abc', blocked=['_abc']) + with support.swap_item(sys.modules, 'abc', abc): + self.assertRaises(OSError, inspect.getsource, abc.ABCMeta) + self.assertRaises(OSError, inspect.getsourcelines, abc.ABCMeta) + # With C acceleration + import abc + try: + src = inspect.getsource(abc.ABCMeta) + lines, lineno = inspect.getsourcelines(abc.ABCMeta) + except OSError: + pass + else: + self.assertEqual(lines[0], ' class ABCMeta(type):\n') + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_decimal(self): + # Pure Python implementation + decimal = import_helper.import_fresh_module('decimal', blocked=['_decimal']) + with support.swap_item(sys.modules, 'decimal', decimal): + src = inspect.getsource(decimal.Decimal) + lines, lineno = inspect.getsourcelines(decimal.Decimal) + self.assertEqual(lines[0], 'class Decimal(object):\n') + self.assertEqual(src.splitlines(True), lines) + class TestGetsourceInteractive(unittest.TestCase): def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; @@ -947,6 +988,11 @@ def test_typeddict(self): self.assertSourceEqual(mod2.td354, 354, 356) self.assertRaises(OSError, inspect.getsource, mod2.td359) + def test_dataclass(self): + self.assertSourceEqual(mod2.dc364, 364, 367) + self.assertRaises(OSError, inspect.getsource, mod2.dc370) + self.assertRaises(OSError, inspect.getsource, mod2.dc371) + class TestBlockComments(GetSourceBase): fodderModule = mod @@ -1010,7 +1056,7 @@ def test_findsource_without_filename(self): self.assertRaises(IOError, inspect.findsource, co) self.assertRaises(IOError, inspect.getsource, co) - def test_findsource_with_out_of_bounds_lineno(self): + def test_findsource_on_func_with_out_of_bounds_lineno(self): mod_len = len(inspect.getsource(mod)) src = '\n' * 2* mod_len + "def f(): pass" co = compile(src, mod.__file__, "exec") @@ -1018,9 +1064,20 @@ def test_findsource_with_out_of_bounds_lineno(self): eval(co, g, l) func = l['f'] self.assertEqual(func.__code__.co_firstlineno, 1+2*mod_len) - with self.assertRaisesRegex(IOError, "lineno is out of bounds"): + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): inspect.findsource(func) + def test_findsource_on_class_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "class A: pass" + co = compile(src, mod.__file__, "exec") + g, l = {'__name__': mod.__name__}, {} + eval(co, g, l) + cls = l['A'] + self.assertEqual(cls.__firstlineno__, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(cls) + def test_getsource_on_method(self): self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst new file mode 100644 index 00000000000000..25b47d5fbaefa5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst @@ -0,0 +1,3 @@ +Setting the :attr:`!__module__` attribute for a class now removes the +``__firstlineno__`` item from the type's dict, so they will no longer be +inconsistent. diff --git a/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst b/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst new file mode 100644 index 00000000000000..e388541f1c2c19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst @@ -0,0 +1,4 @@ +Fix :func:`inspect.getsource` for classes in :mod:`collections.abc` and +:mod:`decimal` (for pure Python implementation) modules. +:func:`inspect.getcomments` now raises OSError instead of IndexError if the +``__firstlineno__`` value for a class is out of bound. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 78f6931dc1d289..e96b69d9d883cc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1444,6 +1444,9 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyType_Modified(type); PyObject *dict = lookup_tp_dict(type); + if (PyDict_Pop(dict, &_Py_ID(__firstlineno__), NULL) < 0) { + return -1; + } return PyDict_SetItem(dict, &_Py_ID(__module__), value); } From 26f4c38f1cfd693a6266052cb526f473b4963104 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 3 Sep 2024 12:22:52 +0300 Subject: [PATCH 2/2] Update Doc/reference/datamodel.rst --- Doc/reference/datamodel.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 2cfadee179ebfb..cf5e83290ed193 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1008,7 +1008,7 @@ Special attributes: :attr:`__firstlineno__` The line number of the first line of the class definition, including decorators. - Setting :attr:`__module__` removes :attr:`__firstlineno__`. + Setting the :attr:`__module__` attribute removes the :attr:`!__firstlineno__` item from the type's dictionary. Class instances