From 1beb6290b2a1b669cc2be8eb0f130d9ae7e5a688 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:25:56 +0100 Subject: [PATCH 01/59] Takes into account the fact that inferring subscript when the node is a class may use the __class_getitem__ method of the current class instead of looking for __getitem__ in the metaclass. --- astroid/scoped_nodes.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index 4c84c5168c..1cc08329d1 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -2617,7 +2617,13 @@ def getitem(self, index, context=None): try: methods = dunder_lookup.lookup(self, "__getitem__") except exceptions.AttributeInferenceError as exc: - raise exceptions.AstroidTypeError(node=self, context=context) from exc + if isinstance(self, ClassDef): + try: + methods = self.getattr("__class_getitem__") + except exceptions.AttributeInferenceError: + raise exceptions.AstroidTypeError(node=self, context=context) from exc + else: + raise exceptions.AstroidTypeError(node=self, context=context) from exc method = methods[0] From 4c85bd0ac0e8c3cbb4db74bc1590044d5c331cb2 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:27:36 +0100 Subject: [PATCH 02/59] OrderedDict in the collections module inherit from dict which is C coded and thus have no metaclass but starting from python3.9 it supports subscripting thanks to the __class_getitem__ method. --- astroid/brain/brain_collections.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index ef6a30a242..c1457f1dd4 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -68,7 +68,7 @@ def __rmul__(self, other): pass""" if PY39: base_deque_class += """ @classmethod - def __class_getitem__(self, item): pass""" + def __class_getitem__(self, item): return cls""" return base_deque_class @@ -77,6 +77,10 @@ def _ordered_dict_mock(): class OrderedDict(dict): def __reversed__(self): return self[::-1] def move_to_end(self, key, last=False): pass""" + if PY39: + base_ordered_dict_class += """ + @classmethod + def __class_getitem__(cls, item): return cls""" return base_ordered_dict_class From 99a078b204ff835cbed373183b92b7959566703d Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:32:53 +0100 Subject: [PATCH 03/59] check_metaclass becomes a static class method because we need it in the class scope. The brain_typing module does not add a ABCMeta_typing class thus there is no need to test it. Moreover it doesn't add neither a __getitem__ to the metaclass --- tests/unittest_brain.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 86cd2e827f..b41675e296 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1211,25 +1211,18 @@ class CustomTD(TypedDict): assert len(typing_module.locals["TypedDict"]) == 1 assert inferred_base == typing_module.locals["TypedDict"][0] + @staticmethod + def check_metaclass_is_abc(node: nodes.ClassDef): + meta = node.metaclass() + assert isinstance(meta, nodes.ClassDef) + assert meta.name == "ABCMeta" + @test_utils.require_version("3.8") def test_typing_alias_type(self): """ Test that the type aliased thanks to typing._alias function are correctly inferred. """ - - def check_metaclass(node: nodes.ClassDef): - meta = node.metaclass() - assert isinstance(meta, nodes.ClassDef) - assert meta.name == "ABCMeta_typing" - assert "ABCMeta" == meta.basenames[0] - assert meta.locals.get("__getitem__") is not None - - abc_meta = next(meta.bases[0].infer()) - assert isinstance(abc_meta, nodes.ClassDef) - assert abc_meta.name == "ABCMeta" - assert abc_meta.locals.get("__getitem__") is None - node = builder.extract_node( """ from typing import TypeVar, MutableSet @@ -1242,7 +1235,7 @@ class Derived1(MutableSet[T]): """ ) inferred = next(node.infer()) - check_metaclass(inferred) + self.check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ From 32d89e2542015b4cb5f74ad8dc75c3984de2c9b6 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:34:15 +0100 Subject: [PATCH 04/59] The brain_typing module does not add anymore _typing suffixed classes in the mro --- tests/unittest_brain.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index b41675e296..fd208e8f73 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1240,7 +1240,6 @@ class Derived1(MutableSet[T]): inferred, [ "Derived1", - "MutableSet_typing", "MutableSet", "Set", "Collection", @@ -1264,7 +1263,6 @@ class Derived2(typing.OrderedDict[int, str]): inferred, [ "Derived2", - "OrderedDict_typing", "OrderedDict", "dict", "object", From 65a8f88be8532dee22497895852bced4211185ee Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:34:53 +0100 Subject: [PATCH 05/59] The OrderedDict class inherits from C coded dict class and thus doesn't have a metaclass. --- tests/unittest_brain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index fd208e8f73..d8734271c1 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1258,7 +1258,7 @@ class Derived2(typing.OrderedDict[int, str]): """ ) inferred = next(node.infer()) - check_metaclass(inferred) + self.assertIsNone(inferred.metaclass()) assertEqualMro( inferred, [ From 0dee7f8dcf47d938915268cf89500f992dd59795 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:36:34 +0100 Subject: [PATCH 06/59] When trying to inherit from typing.Pattern the REPL says : TypeError: type 're.Pattern' is not an acceptable base type --- tests/unittest_brain.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index d8734271c1..db21d019dd 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1269,24 +1269,6 @@ class Derived2(typing.OrderedDict[int, str]): ], ) - node = builder.extract_node( - """ - import typing - class Derived3(typing.Pattern[str]): - pass - """ - ) - inferred = next(node.infer()) - check_metaclass(inferred) - assertEqualMro( - inferred, - [ - "Derived3", - "Pattern", - "object", - ], - ) - @test_utils.require_version("3.8") def test_typing_alias_side_effects(self): """Test that typing._alias changes doesn't have unwanted consequences.""" From 450fae1fb2a03b78a1757d2d262aa4bb388679af Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:38:20 +0100 Subject: [PATCH 07/59] The REPL says that Derived as ABCMeta for metaclass and the mro is Derived => Iterator => Iterable => object --- tests/unittest_brain.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index db21d019dd..999c993dbe 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1282,15 +1282,14 @@ class Derived(collections.abc.Iterator[int]): """ ) inferred = next(node.infer()) - assert inferred.metaclass() is None # Should this be ABCMeta? + self.check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ "Derived", - # Should this be more? - # "Iterator_typing"? - # "Iterator", - # "object", + "Iterator", + "Iterable", + "object", ], ) From eac3ad1c0e4020d93fe79fa3986748ff56935163 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:32:00 +0100 Subject: [PATCH 08/59] Adds comments --- astroid/scoped_nodes.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index 1cc08329d1..d3a13ed752 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -2618,8 +2618,15 @@ def getitem(self, index, context=None): methods = dunder_lookup.lookup(self, "__getitem__") except exceptions.AttributeInferenceError as exc: if isinstance(self, ClassDef): + # subscripting a class definition may be + # achieved thanks to __class_getitem__ method + # which is a classmethod defined in the class + # that supports subscript and not in the metaclass try: methods = self.getattr("__class_getitem__") + # Here it is assumed that the __class_getitem__ node is + # a FunctionDef. One possible improvment would be to deal + # with more generic inference. except exceptions.AttributeInferenceError: raise exceptions.AstroidTypeError(node=self, context=context) from exc else: From 2c711e7e05b09df762e4190e6c5848ea1dd6e37d Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:34:21 +0100 Subject: [PATCH 09/59] Starting with Python39 some collections of the collections.abc module support subscripting thanks to __class_getitem__ method. However the wat it is implemented is not straigthforward and instead of complexifying the way __class_getitem__ is handled inside the getitem method of the ClassDef class, we prefer to hack a bit. --- astroid/brain/brain_collections.py | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index c1457f1dd4..b0b27917ac 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -85,3 +85,42 @@ def __class_getitem__(cls, item): return cls""" astroid.register_module_extender(astroid.MANAGER, "collections", _collections_transform) + + +PY39 = sys.version_info >= (3, 9) + +def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: + """ + Returns True if the node corresponds to a ClassDef of the Collections.abc module that + supports subscripting + + :param node: ClassDef node + """ + if node.qname().startswith('_collections_abc'): + try: + node.getattr('__class_getitem__') + return True + except: + pass + return False + +CLASS_GET_ITEM_TEMPLATE = """ +@classmethod +def __class_getitem__(cls, item): + return cls +""" + +def easy_class_getitem_inference(node, context=None): + # Here __class_getitem__ exists but is quite a mess to infer thus + # put instead an easy inference tip + func_to_add = astroid.extract_node(CLASS_GET_ITEM_TEMPLATE) + node.locals["__class_getitem__"] = [func_to_add] + +if PY39: + # Starting with Python39 some objects of the collection module are subscriptable + # thanks to the __class_getitem__ method but the way it is implemented in + # _collection_abc makes it difficult to infer. (We would have to handle AssignName inference in the + # getitem method of the ClassDef class) Instead we put here a mock of the __class_getitem__ method + astroid.MANAGER.register_transform( + astroid.nodes.ClassDef, easy_class_getitem_inference, _looks_like_subscriptable + ) \ No newline at end of file From 542eb9cc369f6ab4144bb25901ebb9180a013c97 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:36:22 +0100 Subject: [PATCH 10/59] Thanks to __class_getitem__ method there is no need to hack the metaclass --- astroid/brain/brain_typing.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index d699ba451e..2939b0bc75 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -122,31 +122,6 @@ def __getitem__(cls, value): return cls """ -ABC_METACLASS_TEMPLATE = """ -from abc import ABCMeta -ABCMeta -""" - - -@lru_cache() -def create_typing_metaclass(): - #  Needs to mock the __getitem__ class method so that - #  MutableSet[T] is acceptable - func_to_add = extract_node(GET_ITEM_TEMPLATE) - - abc_meta = next(extract_node(ABC_METACLASS_TEMPLATE).infer()) - typing_meta = nodes.ClassDef( - name="ABCMeta_typing", - lineno=abc_meta.lineno, - col_offset=abc_meta.col_offset, - parent=abc_meta.parent, - ) - typing_meta.postinit( - bases=[extract_node(ABC_METACLASS_TEMPLATE)], body=[], decorators=None - ) - typing_meta.locals["__getitem__"] = [func_to_add] - return typing_meta - def _looks_like_typing_alias(node: nodes.Call) -> bool: """ From f5aef79d8854cdcc04c59acce08970caceddac83 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:39:00 +0100 Subject: [PATCH 11/59] SImplifies the inference system for typing objects before python3.9. Before python3.9 the objects of the typing module that are alias of the same objects in the collections.abc module have subscripting possibility thanks to the _GenericAlias metaclass. To mock the subscripting capability we add __class_getitem__ method on those objects. --- astroid/brain/brain_typing.py | 50 +++++++++++++++-------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 2939b0bc75..4cde133559 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -116,9 +116,9 @@ def infer_typedDict( # pylint: disable=invalid-name node.root().locals["TypedDict"] = [class_def] -GET_ITEM_TEMPLATE = """ +CLASS_GETITEM_TEMPLATE = """ @classmethod -def __getitem__(cls, value): +def __class_getitem__(cls, item): return cls """ @@ -154,32 +154,26 @@ def infer_typing_alias( res = next(node.args[0].infer(context=ctx)) if res != astroid.Uninferable and isinstance(res, nodes.ClassDef): - class_def = nodes.ClassDef( - name=f"{res.name}_typing", - lineno=0, - col_offset=0, - parent=res.parent, - ) - class_def.postinit( - bases=[res], - body=res.body, - decorators=res.decorators, - metaclass=create_typing_metaclass(), - ) - return class_def - - if len(node.args) == 2 and isinstance(node.args[0], nodes.Attribute): - class_def = nodes.ClassDef( - name=node.args[0].attrname, - lineno=0, - col_offset=0, - parent=node.parent, - ) - class_def.postinit( - bases=[], body=[], decorators=None, metaclass=create_typing_metaclass() - ) - return class_def - + if not PY39: + # Here the node is a typing object which is an alias toward + # the corresponding object of collection.abc module. + # Before python3.9 there is no subscript allowed for any of the collections.abc objects. + # The subscript ability is given through the typing._GenericAlias class + # which is the metaclass of the typing object but not the metaclass of the inferred + # collections.abc object. + # Thus we fake subscript ability of the collections.abc object + # by mocking the existence of a __class_getitem__ method. + # We can not add `__getitem__` method in the metaclass of the object because + # the metaclass is shared by subscriptable and not subscriptable object + maybe_type_var = node.args[1] + if not (isinstance(maybe_type_var, node_classes.Tuple) and not maybe_type_var.elts): + # The typing object is subscriptable if the second argument of the _alias function + # is a TypeVar or a tuple of TypeVar. We could check the type of the second argument but + # it appears that in the typing module the second argument is only TypeVar or a tuple of TypeVar or empty tuple. + # This last value means the type is not Generic and thus cannot be subscriptable + func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) + res.locals["__class_getitem__"] = [func_to_add] + return res return None From 84c576cdd33f8ec60652f53122672e55f8308b4f Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:40:39 +0100 Subject: [PATCH 12/59] check_metaclass_is_abc become global to be shared among different classes --- tests/unittest_brain.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 999c993dbe..0e879c798b 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1027,6 +1027,10 @@ def test_invalid_type_subscript(self): meth_inf = val_inf.getattr("__class_getitem__")[0] +def check_metaclass_is_abc(node: nodes.ClassDef): + meta = node.metaclass() + assert isinstance(meta, nodes.ClassDef) + assert meta.name == "ABCMeta" @test_utils.require_version("3.6") class TypingBrain(unittest.TestCase): def test_namedtuple_base(self): From 20ca3a111f2d1bea5844f506fbbd20d227a9bdc9 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:41:36 +0100 Subject: [PATCH 13/59] Create a test class dedicated to the Collections brain --- tests/unittest_brain.py | 134 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 0e879c798b..edd89c0d3b 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1031,6 +1031,140 @@ def check_metaclass_is_abc(node: nodes.ClassDef): meta = node.metaclass() assert isinstance(meta, nodes.ClassDef) assert meta.name == "ABCMeta" + + +class CollectionsBrain(unittest.TestCase): + def test_collections_object_not_subscriptable(self): + """Test that unsubscriptable types are detected as so""" + wrong_node = builder.extract_node( + """ + import collections.abc + + collections.abc.Hashable[int] + """ + ) + with self.assertRaises(astroid.exceptions.InferenceError): + next(wrong_node.infer()) + right_node = builder.extract_node( + """ + import collections.abc + + collections.abc.Hashable + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "Hashable", + "object", + ], + ) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + inferred.getattr('__class_getitem__') + + @test_utils.require_version(minver="3.9") + def test_collections_object_subscriptable(self): + """Starting with python39 some object of collections module are subscriptable. Test one of them""" + right_node = builder.extract_node( + """ + import collections.abc + + collections.abc.MutableSet[int] + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "MutableSet", + "Set", + "Collection", + "Sized", + "Iterable", + "Container", + "object" + ] + ) + self.assertIsInstance(inferred.getattr('__class_getitem__')[0], FunctionDef) + + @test_utils.require_version(maxver="3.9") + def test_collections_object_not_yet_subscriptable(self): + """ + Test that unsubscriptable types are detected as so. + Until python39 MutableSet of the collections module is not subscriptable. + """ + wrong_node = builder.extract_node( + """ + import collections.abc + + collections.abc.MutableSet[int] + """ + ) + with self.assertRaises(astroid.exceptions.InferenceError): + next(wrong_node.infer()) + right_node = builder.extract_node( + """ + import collections.abc + + collections.abc.MutableSet + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "MutableSet", + "Set", + "Collection", + "Sized", + "Iterable", + "Container", + "object" + ], + ) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + inferred.getattr('__class_getitem__') + + @test_utils.require_version(minver="3.9") + def test_collections_object_subscriptable_2(self): + """Starting with python39 Iterator in the collection.abc module is subscriptable""" + node = builder.extract_node( + """ + import collections.abc + + class Derived(collections.abc.Iterator[int]): + pass + """ + ) + inferred = next(node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "Derived", + "Iterator", + "Iterable", + "object", + ], + ) + + @test_utils.require_version(maxver="3.8") + def test_collections_object_not_yet_subscriptable_2(self): + """Before python39 Iterator in the collection.abc module is not subscriptable""" + node = builder.extract_node( + """ + import collections.abc + + collections.abc.Iterator[int] + """ + ) + with self.assertRaises(astroid.exceptions.InferenceError): + next(node.infer()) + @test_utils.require_version("3.6") class TypingBrain(unittest.TestCase): def test_namedtuple_base(self): From 66c4c7c3bd3e4ec1f17b30e78a585f2b1858583a Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:42:25 +0100 Subject: [PATCH 14/59] Rewrites and adds test --- tests/unittest_brain.py | 74 +++++++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index edd89c0d3b..23f527219c 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1035,7 +1035,11 @@ def check_metaclass_is_abc(node: nodes.ClassDef): class CollectionsBrain(unittest.TestCase): def test_collections_object_not_subscriptable(self): - """Test that unsubscriptable types are detected as so""" + """ + Test that unsubscriptable types are detected + + Hashable is not subscriptable even with python39 + """ wrong_node = builder.extract_node( """ import collections.abc @@ -1349,17 +1353,12 @@ class CustomTD(TypedDict): assert len(typing_module.locals["TypedDict"]) == 1 assert inferred_base == typing_module.locals["TypedDict"][0] - @staticmethod - def check_metaclass_is_abc(node: nodes.ClassDef): - meta = node.metaclass() - assert isinstance(meta, nodes.ClassDef) - assert meta.name == "ABCMeta" - - @test_utils.require_version("3.8") + @test_utils.require_version(minver="3.7") def test_typing_alias_type(self): """ Test that the type aliased thanks to typing._alias function are correctly inferred. + typing_alias function is introduced with python37 """ node = builder.extract_node( """ @@ -1373,7 +1372,7 @@ class Derived1(MutableSet[T]): """ ) inferred = next(node.infer()) - self.check_metaclass_is_abc(inferred) + check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ @@ -1396,6 +1395,8 @@ class Derived2(typing.OrderedDict[int, str]): """ ) inferred = next(node.infer()) + # OrderedDict has no metaclass because it + # inherits from dict which is C coded self.assertIsNone(inferred.metaclass()) assertEqualMro( inferred, @@ -1407,30 +1408,61 @@ class Derived2(typing.OrderedDict[int, str]): ], ) - @test_utils.require_version("3.8") - def test_typing_alias_side_effects(self): - """Test that typing._alias changes doesn't have unwanted consequences.""" - node = builder.extract_node( + def test_typing_object_not_subscriptable(self): + """Hashable is not subscriptable""" + wrong_node = builder.extract_node( """ import typing - import collections.abc - class Derived(collections.abc.Iterator[int]): - pass + typing.Hashable[int] """ ) - inferred = next(node.infer()) - self.check_metaclass_is_abc(inferred) + with self.assertRaises(astroid.exceptions.InferenceError): + next(wrong_node.infer()) + right_node = builder.extract_node( + """ + import typing + + typing.Hashable + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ - "Derived", - "Iterator", - "Iterable", + "Hashable", "object", ], ) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + inferred.getattr('__class_getitem__') + @test_utils.require_version(minver="3.7") + def test_typing_object_subscriptable(self): + """Test that MutableSet is subscriptable""" + right_node = builder.extract_node( + """ + import typing + + typing.MutableSet[int] + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "MutableSet", + "Set", + "Collection", + "Sized", + "Iterable", + "Container", + "object" + ] + ) + self.assertIsInstance(inferred.getattr('__class_getitem__')[0], FunctionDef) class ReBrainTest(unittest.TestCase): def test_regex_flags(self): From 8c7d34430d1a182d0c27c499b5fe82b175d87e57 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:49:46 +0100 Subject: [PATCH 15/59] Corrects syntax error --- tests/unittest_brain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 23f527219c..bb56efa0a8 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1092,7 +1092,7 @@ def test_collections_object_subscriptable(self): "object" ] ) - self.assertIsInstance(inferred.getattr('__class_getitem__')[0], FunctionDef) + self.assertIsInstance(inferred.getattr('__class_getitem__')[0], nodes.FunctionDef) @test_utils.require_version(maxver="3.9") def test_collections_object_not_yet_subscriptable(self): @@ -1462,7 +1462,7 @@ def test_typing_object_subscriptable(self): "object" ] ) - self.assertIsInstance(inferred.getattr('__class_getitem__')[0], FunctionDef) + self.assertIsInstance(inferred.getattr('__class_getitem__')[0], nodes.FunctionDef) class ReBrainTest(unittest.TestCase): def test_regex_flags(self): From c392180fa0dea20c46c92d000e8988dbe5f90add Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 15:45:25 +0100 Subject: [PATCH 16/59] Deque, defaultdict and OrderedDict are part of the _collections module which is a pure C lib. While letting those class mocks inside collections module is fair for astroid it leds to pylint acceptance tests fail. --- astroid/brain/brain_collections.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index b0b27917ac..fd1504c976 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -84,7 +84,16 @@ def __class_getitem__(cls, item): return cls""" return base_ordered_dict_class -astroid.register_module_extender(astroid.MANAGER, "collections", _collections_transform) +def _collections_module_properties(node, context=None): + """ + Adds a path to a fictive file as the _collections module is a pure C lib. + """ + node.file = "/tmp/unknown" + return node + + +astroid.MANAGER.register_transform(astroid.Module, _collections_module_properties, lambda n: n.name == "_collections") +astroid.register_module_extender(astroid.MANAGER, "_collections", _collections_transform) PY39 = sys.version_info >= (3, 9) From b0d13d15a0d331e9f5a829234ae625d3ab26e322 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 15:49:35 +0100 Subject: [PATCH 17/59] Formatting according to black --- astroid/brain/brain_collections.py | 28 +++++++++++++++--------- astroid/brain/brain_typing.py | 21 ++++++++++-------- astroid/scoped_nodes.py | 14 ++++++------ tests/unittest_brain.py | 34 ++++++++++++++++++------------ 4 files changed, 58 insertions(+), 39 deletions(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index fd1504c976..5ac0cf73a9 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -92,12 +92,17 @@ def _collections_module_properties(node, context=None): return node -astroid.MANAGER.register_transform(astroid.Module, _collections_module_properties, lambda n: n.name == "_collections") -astroid.register_module_extender(astroid.MANAGER, "_collections", _collections_transform) +astroid.MANAGER.register_transform( + astroid.Module, _collections_module_properties, lambda n: n.name == "_collections" +) +astroid.register_module_extender( + astroid.MANAGER, "_collections", _collections_transform +) PY39 = sys.version_info >= (3, 9) + def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: """ Returns True if the node corresponds to a ClassDef of the Collections.abc module that @@ -105,31 +110,34 @@ def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: :param node: ClassDef node """ - if node.qname().startswith('_collections_abc'): + if node.qname().startswith("_collections_abc"): try: - node.getattr('__class_getitem__') + node.getattr("__class_getitem__") return True except: pass return False + CLASS_GET_ITEM_TEMPLATE = """ @classmethod def __class_getitem__(cls, item): return cls """ + def easy_class_getitem_inference(node, context=None): - # Here __class_getitem__ exists but is quite a mess to infer thus - # put instead an easy inference tip + #  Here __class_getitem__ exists but is quite a mess to infer thus + #  put instead an easy inference tip func_to_add = astroid.extract_node(CLASS_GET_ITEM_TEMPLATE) node.locals["__class_getitem__"] = [func_to_add] + if PY39: - # Starting with Python39 some objects of the collection module are subscriptable - # thanks to the __class_getitem__ method but the way it is implemented in - # _collection_abc makes it difficult to infer. (We would have to handle AssignName inference in the + #  Starting with Python39 some objects of the collection module are subscriptable + #  thanks to the __class_getitem__ method but the way it is implemented in + #  _collection_abc makes it difficult to infer. (We would have to handle AssignName inference in the # getitem method of the ClassDef class) Instead we put here a mock of the __class_getitem__ method astroid.MANAGER.register_transform( astroid.nodes.ClassDef, easy_class_getitem_inference, _looks_like_subscriptable - ) \ No newline at end of file + ) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 4cde133559..af4b7e8b18 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -155,21 +155,24 @@ def infer_typing_alias( if res != astroid.Uninferable and isinstance(res, nodes.ClassDef): if not PY39: - # Here the node is a typing object which is an alias toward + #  Here the node is a typing object which is an alias toward # the corresponding object of collection.abc module. # Before python3.9 there is no subscript allowed for any of the collections.abc objects. # The subscript ability is given through the typing._GenericAlias class - # which is the metaclass of the typing object but not the metaclass of the inferred - # collections.abc object. + #  which is the metaclass of the typing object but not the metaclass of the inferred + #  collections.abc object. # Thus we fake subscript ability of the collections.abc object - # by mocking the existence of a __class_getitem__ method. - # We can not add `__getitem__` method in the metaclass of the object because - # the metaclass is shared by subscriptable and not subscriptable object + #  by mocking the existence of a __class_getitem__ method. + #  We can not add `__getitem__` method in the metaclass of the object because + #  the metaclass is shared by subscriptable and not subscriptable object maybe_type_var = node.args[1] - if not (isinstance(maybe_type_var, node_classes.Tuple) and not maybe_type_var.elts): - # The typing object is subscriptable if the second argument of the _alias function + if not ( + isinstance(maybe_type_var, node_classes.Tuple) + and not maybe_type_var.elts + ): + # The typing object is subscriptable if the second argument of the _alias function # is a TypeVar or a tuple of TypeVar. We could check the type of the second argument but - # it appears that in the typing module the second argument is only TypeVar or a tuple of TypeVar or empty tuple. + #  it appears that in the typing module the second argument is only TypeVar or a tuple of TypeVar or empty tuple. # This last value means the type is not Generic and thus cannot be subscriptable func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) res.locals["__class_getitem__"] = [func_to_add] diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index d3a13ed752..261ed99bba 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -2618,17 +2618,19 @@ def getitem(self, index, context=None): methods = dunder_lookup.lookup(self, "__getitem__") except exceptions.AttributeInferenceError as exc: if isinstance(self, ClassDef): - # subscripting a class definition may be - # achieved thanks to __class_getitem__ method - # which is a classmethod defined in the class - # that supports subscript and not in the metaclass + # subscripting a class definition may be + #  achieved thanks to __class_getitem__ method + #  which is a classmethod defined in the class + #  that supports subscript and not in the metaclass try: methods = self.getattr("__class_getitem__") # Here it is assumed that the __class_getitem__ node is - # a FunctionDef. One possible improvment would be to deal + #  a FunctionDef. One possible improvment would be to deal # with more generic inference. except exceptions.AttributeInferenceError: - raise exceptions.AstroidTypeError(node=self, context=context) from exc + raise exceptions.AstroidTypeError( + node=self, context=context + ) from exc else: raise exceptions.AstroidTypeError(node=self, context=context) from exc diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index bb56efa0a8..bd9d924634 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1066,7 +1066,7 @@ def test_collections_object_not_subscriptable(self): ], ) with self.assertRaises(astroid.exceptions.AttributeInferenceError): - inferred.getattr('__class_getitem__') + inferred.getattr("__class_getitem__") @test_utils.require_version(minver="3.9") def test_collections_object_subscriptable(self): @@ -1089,10 +1089,12 @@ def test_collections_object_subscriptable(self): "Sized", "Iterable", "Container", - "object" - ] + "object", + ], + ) + self.assertIsInstance( + inferred.getattr("__class_getitem__")[0], nodes.FunctionDef ) - self.assertIsInstance(inferred.getattr('__class_getitem__')[0], nodes.FunctionDef) @test_utils.require_version(maxver="3.9") def test_collections_object_not_yet_subscriptable(self): @@ -1127,11 +1129,11 @@ def test_collections_object_not_yet_subscriptable(self): "Sized", "Iterable", "Container", - "object" + "object", ], ) with self.assertRaises(astroid.exceptions.AttributeInferenceError): - inferred.getattr('__class_getitem__') + inferred.getattr("__class_getitem__") @test_utils.require_version(minver="3.9") def test_collections_object_subscriptable_2(self): @@ -1169,6 +1171,7 @@ def test_collections_object_not_yet_subscriptable_2(self): with self.assertRaises(astroid.exceptions.InferenceError): next(node.infer()) + @test_utils.require_version("3.6") class TypingBrain(unittest.TestCase): def test_namedtuple_base(self): @@ -1353,7 +1356,7 @@ class CustomTD(TypedDict): assert len(typing_module.locals["TypedDict"]) == 1 assert inferred_base == typing_module.locals["TypedDict"][0] - @test_utils.require_version(minver="3.7") + @test_utils.require_version(minver="3.7") def test_typing_alias_type(self): """ Test that the type aliased thanks to typing._alias function are @@ -1395,8 +1398,8 @@ class Derived2(typing.OrderedDict[int, str]): """ ) inferred = next(node.infer()) - # OrderedDict has no metaclass because it - # inherits from dict which is C coded + #  OrderedDict has no metaclass because it + #  inherits from dict which is C coded self.assertIsNone(inferred.metaclass()) assertEqualMro( inferred, @@ -1436,9 +1439,9 @@ def test_typing_object_not_subscriptable(self): ], ) with self.assertRaises(astroid.exceptions.AttributeInferenceError): - inferred.getattr('__class_getitem__') + inferred.getattr("__class_getitem__") - @test_utils.require_version(minver="3.7") + @test_utils.require_version(minver="3.7") def test_typing_object_subscriptable(self): """Test that MutableSet is subscriptable""" right_node = builder.extract_node( @@ -1459,10 +1462,13 @@ def test_typing_object_subscriptable(self): "Sized", "Iterable", "Container", - "object" - ] + "object", + ], + ) + self.assertIsInstance( + inferred.getattr("__class_getitem__")[0], nodes.FunctionDef ) - self.assertIsInstance(inferred.getattr('__class_getitem__')[0], nodes.FunctionDef) + class ReBrainTest(unittest.TestCase): def test_regex_flags(self): From ab2667530e813a88e8dc0631e94591cb80c6a971 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 15:53:45 +0100 Subject: [PATCH 18/59] Adds two entries --- ChangeLog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 1c66a938ef..eb5f5acd55 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,12 @@ What's New in astroid 2.5.2? ============================ Release Date: TBA +* Takes into account the fact that subscript inferring for a ClassDef may involve __class_getitem__ method + +* Reworks the `collections` and `typing` brain so that `pylint`s acceptance tests are fine. + + Closes PyCQA/pylint#4206 + * Detects `import numpy` as a valid `numpy` import. Closes PyCQA/pylint#3974 From 230fee40c4738ee0bda29c0937e7fab74635de22 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 18:20:53 +0100 Subject: [PATCH 19/59] Extends the filter to determine what is subscriptable to include OrderedDict --- astroid/brain/brain_collections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 5ac0cf73a9..40e0da78e1 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -110,7 +110,7 @@ def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: :param node: ClassDef node """ - if node.qname().startswith("_collections_abc"): + if node.qname().startswith("_collections") or node.qname().startswith("collections"): try: node.getattr("__class_getitem__") return True From 79b502c691e4bcd5e2103b2b4c44f6f770a7382f Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 18:27:02 +0100 Subject: [PATCH 20/59] Formatting according to black --- astroid/brain/brain_collections.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 40e0da78e1..3c8d85f82b 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -110,7 +110,9 @@ def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: :param node: ClassDef node """ - if node.qname().startswith("_collections") or node.qname().startswith("collections"): + if node.qname().startswith("_collections") or node.qname().startswith( + "collections" + ): try: node.getattr("__class_getitem__") return True From 91ce86457a3810e3cd5f7fd5c5b4438717a31324 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:25:56 +0100 Subject: [PATCH 21/59] Takes into account the fact that inferring subscript when the node is a class may use the __class_getitem__ method of the current class instead of looking for __getitem__ in the metaclass. --- astroid/scoped_nodes.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index 41aa9e1bd9..c7f7f3b2da 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -2617,7 +2617,13 @@ def getitem(self, index, context=None): try: methods = dunder_lookup.lookup(self, "__getitem__") except exceptions.AttributeInferenceError as exc: - raise exceptions.AstroidTypeError(node=self, context=context) from exc + if isinstance(self, ClassDef): + try: + methods = self.getattr("__class_getitem__") + except exceptions.AttributeInferenceError: + raise exceptions.AstroidTypeError(node=self, context=context) from exc + else: + raise exceptions.AstroidTypeError(node=self, context=context) from exc method = methods[0] From 136169b9d9f56505217cf8a0571c54c8b7770b3d Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:27:36 +0100 Subject: [PATCH 22/59] OrderedDict in the collections module inherit from dict which is C coded and thus have no metaclass but starting from python3.9 it supports subscripting thanks to the __class_getitem__ method. --- astroid/brain/brain_collections.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index ef6a30a242..c1457f1dd4 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -68,7 +68,7 @@ def __rmul__(self, other): pass""" if PY39: base_deque_class += """ @classmethod - def __class_getitem__(self, item): pass""" + def __class_getitem__(self, item): return cls""" return base_deque_class @@ -77,6 +77,10 @@ def _ordered_dict_mock(): class OrderedDict(dict): def __reversed__(self): return self[::-1] def move_to_end(self, key, last=False): pass""" + if PY39: + base_ordered_dict_class += """ + @classmethod + def __class_getitem__(cls, item): return cls""" return base_ordered_dict_class From 4e4e27f114a35f811d34a3a1292449b0f65bbd9c Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:32:53 +0100 Subject: [PATCH 23/59] check_metaclass becomes a static class method because we need it in the class scope. The brain_typing module does not add a ABCMeta_typing class thus there is no need to test it. Moreover it doesn't add neither a __getitem__ to the metaclass --- tests/unittest_brain.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 86cd2e827f..b41675e296 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1211,25 +1211,18 @@ class CustomTD(TypedDict): assert len(typing_module.locals["TypedDict"]) == 1 assert inferred_base == typing_module.locals["TypedDict"][0] + @staticmethod + def check_metaclass_is_abc(node: nodes.ClassDef): + meta = node.metaclass() + assert isinstance(meta, nodes.ClassDef) + assert meta.name == "ABCMeta" + @test_utils.require_version("3.8") def test_typing_alias_type(self): """ Test that the type aliased thanks to typing._alias function are correctly inferred. """ - - def check_metaclass(node: nodes.ClassDef): - meta = node.metaclass() - assert isinstance(meta, nodes.ClassDef) - assert meta.name == "ABCMeta_typing" - assert "ABCMeta" == meta.basenames[0] - assert meta.locals.get("__getitem__") is not None - - abc_meta = next(meta.bases[0].infer()) - assert isinstance(abc_meta, nodes.ClassDef) - assert abc_meta.name == "ABCMeta" - assert abc_meta.locals.get("__getitem__") is None - node = builder.extract_node( """ from typing import TypeVar, MutableSet @@ -1242,7 +1235,7 @@ class Derived1(MutableSet[T]): """ ) inferred = next(node.infer()) - check_metaclass(inferred) + self.check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ From 324f02c147bb2f9ca2604ee5357754d62e358300 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:34:15 +0100 Subject: [PATCH 24/59] The brain_typing module does not add anymore _typing suffixed classes in the mro --- tests/unittest_brain.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index b41675e296..fd208e8f73 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1240,7 +1240,6 @@ class Derived1(MutableSet[T]): inferred, [ "Derived1", - "MutableSet_typing", "MutableSet", "Set", "Collection", @@ -1264,7 +1263,6 @@ class Derived2(typing.OrderedDict[int, str]): inferred, [ "Derived2", - "OrderedDict_typing", "OrderedDict", "dict", "object", From 3babaf4190913effa03c9505ec41e1f6532481b6 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:34:53 +0100 Subject: [PATCH 25/59] The OrderedDict class inherits from C coded dict class and thus doesn't have a metaclass. --- tests/unittest_brain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index fd208e8f73..d8734271c1 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1258,7 +1258,7 @@ class Derived2(typing.OrderedDict[int, str]): """ ) inferred = next(node.infer()) - check_metaclass(inferred) + self.assertIsNone(inferred.metaclass()) assertEqualMro( inferred, [ From a1f1dd6085495005a324b4691e12befe0d79a9d1 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:36:34 +0100 Subject: [PATCH 26/59] When trying to inherit from typing.Pattern the REPL says : TypeError: type 're.Pattern' is not an acceptable base type --- tests/unittest_brain.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index d8734271c1..db21d019dd 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1269,24 +1269,6 @@ class Derived2(typing.OrderedDict[int, str]): ], ) - node = builder.extract_node( - """ - import typing - class Derived3(typing.Pattern[str]): - pass - """ - ) - inferred = next(node.infer()) - check_metaclass(inferred) - assertEqualMro( - inferred, - [ - "Derived3", - "Pattern", - "object", - ], - ) - @test_utils.require_version("3.8") def test_typing_alias_side_effects(self): """Test that typing._alias changes doesn't have unwanted consequences.""" From 92a942267a290e44c120b39e028b6408dc1fd54d Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 13 Mar 2021 18:38:20 +0100 Subject: [PATCH 27/59] The REPL says that Derived as ABCMeta for metaclass and the mro is Derived => Iterator => Iterable => object --- tests/unittest_brain.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index db21d019dd..999c993dbe 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1282,15 +1282,14 @@ class Derived(collections.abc.Iterator[int]): """ ) inferred = next(node.infer()) - assert inferred.metaclass() is None # Should this be ABCMeta? + self.check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ "Derived", - # Should this be more? - # "Iterator_typing"? - # "Iterator", - # "object", + "Iterator", + "Iterable", + "object", ], ) From 7afcd62f368e12faa24de8e0faf16ecead0967c1 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:32:00 +0100 Subject: [PATCH 28/59] Adds comments --- astroid/scoped_nodes.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index c7f7f3b2da..fb802ba1fb 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -2618,8 +2618,15 @@ def getitem(self, index, context=None): methods = dunder_lookup.lookup(self, "__getitem__") except exceptions.AttributeInferenceError as exc: if isinstance(self, ClassDef): + # subscripting a class definition may be + # achieved thanks to __class_getitem__ method + # which is a classmethod defined in the class + # that supports subscript and not in the metaclass try: methods = self.getattr("__class_getitem__") + # Here it is assumed that the __class_getitem__ node is + # a FunctionDef. One possible improvment would be to deal + # with more generic inference. except exceptions.AttributeInferenceError: raise exceptions.AstroidTypeError(node=self, context=context) from exc else: From 7b16e00d80f98530613b6eac5d559d3fb0338f28 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:34:21 +0100 Subject: [PATCH 29/59] Starting with Python39 some collections of the collections.abc module support subscripting thanks to __class_getitem__ method. However the wat it is implemented is not straigthforward and instead of complexifying the way __class_getitem__ is handled inside the getitem method of the ClassDef class, we prefer to hack a bit. --- astroid/brain/brain_collections.py | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index c1457f1dd4..b0b27917ac 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -85,3 +85,42 @@ def __class_getitem__(cls, item): return cls""" astroid.register_module_extender(astroid.MANAGER, "collections", _collections_transform) + + +PY39 = sys.version_info >= (3, 9) + +def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: + """ + Returns True if the node corresponds to a ClassDef of the Collections.abc module that + supports subscripting + + :param node: ClassDef node + """ + if node.qname().startswith('_collections_abc'): + try: + node.getattr('__class_getitem__') + return True + except: + pass + return False + +CLASS_GET_ITEM_TEMPLATE = """ +@classmethod +def __class_getitem__(cls, item): + return cls +""" + +def easy_class_getitem_inference(node, context=None): + # Here __class_getitem__ exists but is quite a mess to infer thus + # put instead an easy inference tip + func_to_add = astroid.extract_node(CLASS_GET_ITEM_TEMPLATE) + node.locals["__class_getitem__"] = [func_to_add] + +if PY39: + # Starting with Python39 some objects of the collection module are subscriptable + # thanks to the __class_getitem__ method but the way it is implemented in + # _collection_abc makes it difficult to infer. (We would have to handle AssignName inference in the + # getitem method of the ClassDef class) Instead we put here a mock of the __class_getitem__ method + astroid.MANAGER.register_transform( + astroid.nodes.ClassDef, easy_class_getitem_inference, _looks_like_subscriptable + ) \ No newline at end of file From 5b5683630ea89ce0e86742edb255bc1695e1058b Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:36:22 +0100 Subject: [PATCH 30/59] Thanks to __class_getitem__ method there is no need to hack the metaclass --- astroid/brain/brain_typing.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index d699ba451e..2939b0bc75 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -122,31 +122,6 @@ def __getitem__(cls, value): return cls """ -ABC_METACLASS_TEMPLATE = """ -from abc import ABCMeta -ABCMeta -""" - - -@lru_cache() -def create_typing_metaclass(): - #  Needs to mock the __getitem__ class method so that - #  MutableSet[T] is acceptable - func_to_add = extract_node(GET_ITEM_TEMPLATE) - - abc_meta = next(extract_node(ABC_METACLASS_TEMPLATE).infer()) - typing_meta = nodes.ClassDef( - name="ABCMeta_typing", - lineno=abc_meta.lineno, - col_offset=abc_meta.col_offset, - parent=abc_meta.parent, - ) - typing_meta.postinit( - bases=[extract_node(ABC_METACLASS_TEMPLATE)], body=[], decorators=None - ) - typing_meta.locals["__getitem__"] = [func_to_add] - return typing_meta - def _looks_like_typing_alias(node: nodes.Call) -> bool: """ From 6f4328aea303a955947f463493347bffd2f5eb86 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:39:00 +0100 Subject: [PATCH 31/59] SImplifies the inference system for typing objects before python3.9. Before python3.9 the objects of the typing module that are alias of the same objects in the collections.abc module have subscripting possibility thanks to the _GenericAlias metaclass. To mock the subscripting capability we add __class_getitem__ method on those objects. --- astroid/brain/brain_typing.py | 50 +++++++++++++++-------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 2939b0bc75..4cde133559 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -116,9 +116,9 @@ def infer_typedDict( # pylint: disable=invalid-name node.root().locals["TypedDict"] = [class_def] -GET_ITEM_TEMPLATE = """ +CLASS_GETITEM_TEMPLATE = """ @classmethod -def __getitem__(cls, value): +def __class_getitem__(cls, item): return cls """ @@ -154,32 +154,26 @@ def infer_typing_alias( res = next(node.args[0].infer(context=ctx)) if res != astroid.Uninferable and isinstance(res, nodes.ClassDef): - class_def = nodes.ClassDef( - name=f"{res.name}_typing", - lineno=0, - col_offset=0, - parent=res.parent, - ) - class_def.postinit( - bases=[res], - body=res.body, - decorators=res.decorators, - metaclass=create_typing_metaclass(), - ) - return class_def - - if len(node.args) == 2 and isinstance(node.args[0], nodes.Attribute): - class_def = nodes.ClassDef( - name=node.args[0].attrname, - lineno=0, - col_offset=0, - parent=node.parent, - ) - class_def.postinit( - bases=[], body=[], decorators=None, metaclass=create_typing_metaclass() - ) - return class_def - + if not PY39: + # Here the node is a typing object which is an alias toward + # the corresponding object of collection.abc module. + # Before python3.9 there is no subscript allowed for any of the collections.abc objects. + # The subscript ability is given through the typing._GenericAlias class + # which is the metaclass of the typing object but not the metaclass of the inferred + # collections.abc object. + # Thus we fake subscript ability of the collections.abc object + # by mocking the existence of a __class_getitem__ method. + # We can not add `__getitem__` method in the metaclass of the object because + # the metaclass is shared by subscriptable and not subscriptable object + maybe_type_var = node.args[1] + if not (isinstance(maybe_type_var, node_classes.Tuple) and not maybe_type_var.elts): + # The typing object is subscriptable if the second argument of the _alias function + # is a TypeVar or a tuple of TypeVar. We could check the type of the second argument but + # it appears that in the typing module the second argument is only TypeVar or a tuple of TypeVar or empty tuple. + # This last value means the type is not Generic and thus cannot be subscriptable + func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) + res.locals["__class_getitem__"] = [func_to_add] + return res return None From a38e82d7ddcea3d327a7472b29c83b1087b88b27 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:40:39 +0100 Subject: [PATCH 32/59] check_metaclass_is_abc become global to be shared among different classes --- tests/unittest_brain.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 999c993dbe..0e879c798b 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1027,6 +1027,10 @@ def test_invalid_type_subscript(self): meth_inf = val_inf.getattr("__class_getitem__")[0] +def check_metaclass_is_abc(node: nodes.ClassDef): + meta = node.metaclass() + assert isinstance(meta, nodes.ClassDef) + assert meta.name == "ABCMeta" @test_utils.require_version("3.6") class TypingBrain(unittest.TestCase): def test_namedtuple_base(self): From 121a3db5ba76312a4e631c97a01add6278ce7d9a Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:41:36 +0100 Subject: [PATCH 33/59] Create a test class dedicated to the Collections brain --- tests/unittest_brain.py | 134 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 0e879c798b..edd89c0d3b 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1031,6 +1031,140 @@ def check_metaclass_is_abc(node: nodes.ClassDef): meta = node.metaclass() assert isinstance(meta, nodes.ClassDef) assert meta.name == "ABCMeta" + + +class CollectionsBrain(unittest.TestCase): + def test_collections_object_not_subscriptable(self): + """Test that unsubscriptable types are detected as so""" + wrong_node = builder.extract_node( + """ + import collections.abc + + collections.abc.Hashable[int] + """ + ) + with self.assertRaises(astroid.exceptions.InferenceError): + next(wrong_node.infer()) + right_node = builder.extract_node( + """ + import collections.abc + + collections.abc.Hashable + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "Hashable", + "object", + ], + ) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + inferred.getattr('__class_getitem__') + + @test_utils.require_version(minver="3.9") + def test_collections_object_subscriptable(self): + """Starting with python39 some object of collections module are subscriptable. Test one of them""" + right_node = builder.extract_node( + """ + import collections.abc + + collections.abc.MutableSet[int] + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "MutableSet", + "Set", + "Collection", + "Sized", + "Iterable", + "Container", + "object" + ] + ) + self.assertIsInstance(inferred.getattr('__class_getitem__')[0], FunctionDef) + + @test_utils.require_version(maxver="3.9") + def test_collections_object_not_yet_subscriptable(self): + """ + Test that unsubscriptable types are detected as so. + Until python39 MutableSet of the collections module is not subscriptable. + """ + wrong_node = builder.extract_node( + """ + import collections.abc + + collections.abc.MutableSet[int] + """ + ) + with self.assertRaises(astroid.exceptions.InferenceError): + next(wrong_node.infer()) + right_node = builder.extract_node( + """ + import collections.abc + + collections.abc.MutableSet + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "MutableSet", + "Set", + "Collection", + "Sized", + "Iterable", + "Container", + "object" + ], + ) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + inferred.getattr('__class_getitem__') + + @test_utils.require_version(minver="3.9") + def test_collections_object_subscriptable_2(self): + """Starting with python39 Iterator in the collection.abc module is subscriptable""" + node = builder.extract_node( + """ + import collections.abc + + class Derived(collections.abc.Iterator[int]): + pass + """ + ) + inferred = next(node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "Derived", + "Iterator", + "Iterable", + "object", + ], + ) + + @test_utils.require_version(maxver="3.8") + def test_collections_object_not_yet_subscriptable_2(self): + """Before python39 Iterator in the collection.abc module is not subscriptable""" + node = builder.extract_node( + """ + import collections.abc + + collections.abc.Iterator[int] + """ + ) + with self.assertRaises(astroid.exceptions.InferenceError): + next(node.infer()) + @test_utils.require_version("3.6") class TypingBrain(unittest.TestCase): def test_namedtuple_base(self): From 0654a69dc4122651236436831706f882691cdfde Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:42:25 +0100 Subject: [PATCH 34/59] Rewrites and adds test --- tests/unittest_brain.py | 74 +++++++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index edd89c0d3b..23f527219c 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1035,7 +1035,11 @@ def check_metaclass_is_abc(node: nodes.ClassDef): class CollectionsBrain(unittest.TestCase): def test_collections_object_not_subscriptable(self): - """Test that unsubscriptable types are detected as so""" + """ + Test that unsubscriptable types are detected + + Hashable is not subscriptable even with python39 + """ wrong_node = builder.extract_node( """ import collections.abc @@ -1349,17 +1353,12 @@ class CustomTD(TypedDict): assert len(typing_module.locals["TypedDict"]) == 1 assert inferred_base == typing_module.locals["TypedDict"][0] - @staticmethod - def check_metaclass_is_abc(node: nodes.ClassDef): - meta = node.metaclass() - assert isinstance(meta, nodes.ClassDef) - assert meta.name == "ABCMeta" - - @test_utils.require_version("3.8") + @test_utils.require_version(minver="3.7") def test_typing_alias_type(self): """ Test that the type aliased thanks to typing._alias function are correctly inferred. + typing_alias function is introduced with python37 """ node = builder.extract_node( """ @@ -1373,7 +1372,7 @@ class Derived1(MutableSet[T]): """ ) inferred = next(node.infer()) - self.check_metaclass_is_abc(inferred) + check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ @@ -1396,6 +1395,8 @@ class Derived2(typing.OrderedDict[int, str]): """ ) inferred = next(node.infer()) + # OrderedDict has no metaclass because it + # inherits from dict which is C coded self.assertIsNone(inferred.metaclass()) assertEqualMro( inferred, @@ -1407,30 +1408,61 @@ class Derived2(typing.OrderedDict[int, str]): ], ) - @test_utils.require_version("3.8") - def test_typing_alias_side_effects(self): - """Test that typing._alias changes doesn't have unwanted consequences.""" - node = builder.extract_node( + def test_typing_object_not_subscriptable(self): + """Hashable is not subscriptable""" + wrong_node = builder.extract_node( """ import typing - import collections.abc - class Derived(collections.abc.Iterator[int]): - pass + typing.Hashable[int] """ ) - inferred = next(node.infer()) - self.check_metaclass_is_abc(inferred) + with self.assertRaises(astroid.exceptions.InferenceError): + next(wrong_node.infer()) + right_node = builder.extract_node( + """ + import typing + + typing.Hashable + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ - "Derived", - "Iterator", - "Iterable", + "Hashable", "object", ], ) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + inferred.getattr('__class_getitem__') + @test_utils.require_version(minver="3.7") + def test_typing_object_subscriptable(self): + """Test that MutableSet is subscriptable""" + right_node = builder.extract_node( + """ + import typing + + typing.MutableSet[int] + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "MutableSet", + "Set", + "Collection", + "Sized", + "Iterable", + "Container", + "object" + ] + ) + self.assertIsInstance(inferred.getattr('__class_getitem__')[0], FunctionDef) class ReBrainTest(unittest.TestCase): def test_regex_flags(self): From 2e382defdc5143cb04b5cb0a1da44ba758a53b2c Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 13:49:46 +0100 Subject: [PATCH 35/59] Corrects syntax error --- tests/unittest_brain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 23f527219c..bb56efa0a8 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1092,7 +1092,7 @@ def test_collections_object_subscriptable(self): "object" ] ) - self.assertIsInstance(inferred.getattr('__class_getitem__')[0], FunctionDef) + self.assertIsInstance(inferred.getattr('__class_getitem__')[0], nodes.FunctionDef) @test_utils.require_version(maxver="3.9") def test_collections_object_not_yet_subscriptable(self): @@ -1462,7 +1462,7 @@ def test_typing_object_subscriptable(self): "object" ] ) - self.assertIsInstance(inferred.getattr('__class_getitem__')[0], FunctionDef) + self.assertIsInstance(inferred.getattr('__class_getitem__')[0], nodes.FunctionDef) class ReBrainTest(unittest.TestCase): def test_regex_flags(self): From 86bd75a069a42351caa746a9b173615a18501b2e Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 15:45:25 +0100 Subject: [PATCH 36/59] Deque, defaultdict and OrderedDict are part of the _collections module which is a pure C lib. While letting those class mocks inside collections module is fair for astroid it leds to pylint acceptance tests fail. --- astroid/brain/brain_collections.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index b0b27917ac..fd1504c976 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -84,7 +84,16 @@ def __class_getitem__(cls, item): return cls""" return base_ordered_dict_class -astroid.register_module_extender(astroid.MANAGER, "collections", _collections_transform) +def _collections_module_properties(node, context=None): + """ + Adds a path to a fictive file as the _collections module is a pure C lib. + """ + node.file = "/tmp/unknown" + return node + + +astroid.MANAGER.register_transform(astroid.Module, _collections_module_properties, lambda n: n.name == "_collections") +astroid.register_module_extender(astroid.MANAGER, "_collections", _collections_transform) PY39 = sys.version_info >= (3, 9) From eb87d799793d000d222812660ae113b2b6f633a3 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 15:49:35 +0100 Subject: [PATCH 37/59] Formatting according to black --- astroid/brain/brain_collections.py | 28 +++++++++++++++--------- astroid/brain/brain_typing.py | 21 ++++++++++-------- astroid/scoped_nodes.py | 14 ++++++------ tests/unittest_brain.py | 34 ++++++++++++++++++------------ 4 files changed, 58 insertions(+), 39 deletions(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index fd1504c976..5ac0cf73a9 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -92,12 +92,17 @@ def _collections_module_properties(node, context=None): return node -astroid.MANAGER.register_transform(astroid.Module, _collections_module_properties, lambda n: n.name == "_collections") -astroid.register_module_extender(astroid.MANAGER, "_collections", _collections_transform) +astroid.MANAGER.register_transform( + astroid.Module, _collections_module_properties, lambda n: n.name == "_collections" +) +astroid.register_module_extender( + astroid.MANAGER, "_collections", _collections_transform +) PY39 = sys.version_info >= (3, 9) + def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: """ Returns True if the node corresponds to a ClassDef of the Collections.abc module that @@ -105,31 +110,34 @@ def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: :param node: ClassDef node """ - if node.qname().startswith('_collections_abc'): + if node.qname().startswith("_collections_abc"): try: - node.getattr('__class_getitem__') + node.getattr("__class_getitem__") return True except: pass return False + CLASS_GET_ITEM_TEMPLATE = """ @classmethod def __class_getitem__(cls, item): return cls """ + def easy_class_getitem_inference(node, context=None): - # Here __class_getitem__ exists but is quite a mess to infer thus - # put instead an easy inference tip + #  Here __class_getitem__ exists but is quite a mess to infer thus + #  put instead an easy inference tip func_to_add = astroid.extract_node(CLASS_GET_ITEM_TEMPLATE) node.locals["__class_getitem__"] = [func_to_add] + if PY39: - # Starting with Python39 some objects of the collection module are subscriptable - # thanks to the __class_getitem__ method but the way it is implemented in - # _collection_abc makes it difficult to infer. (We would have to handle AssignName inference in the + #  Starting with Python39 some objects of the collection module are subscriptable + #  thanks to the __class_getitem__ method but the way it is implemented in + #  _collection_abc makes it difficult to infer. (We would have to handle AssignName inference in the # getitem method of the ClassDef class) Instead we put here a mock of the __class_getitem__ method astroid.MANAGER.register_transform( astroid.nodes.ClassDef, easy_class_getitem_inference, _looks_like_subscriptable - ) \ No newline at end of file + ) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 4cde133559..af4b7e8b18 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -155,21 +155,24 @@ def infer_typing_alias( if res != astroid.Uninferable and isinstance(res, nodes.ClassDef): if not PY39: - # Here the node is a typing object which is an alias toward + #  Here the node is a typing object which is an alias toward # the corresponding object of collection.abc module. # Before python3.9 there is no subscript allowed for any of the collections.abc objects. # The subscript ability is given through the typing._GenericAlias class - # which is the metaclass of the typing object but not the metaclass of the inferred - # collections.abc object. + #  which is the metaclass of the typing object but not the metaclass of the inferred + #  collections.abc object. # Thus we fake subscript ability of the collections.abc object - # by mocking the existence of a __class_getitem__ method. - # We can not add `__getitem__` method in the metaclass of the object because - # the metaclass is shared by subscriptable and not subscriptable object + #  by mocking the existence of a __class_getitem__ method. + #  We can not add `__getitem__` method in the metaclass of the object because + #  the metaclass is shared by subscriptable and not subscriptable object maybe_type_var = node.args[1] - if not (isinstance(maybe_type_var, node_classes.Tuple) and not maybe_type_var.elts): - # The typing object is subscriptable if the second argument of the _alias function + if not ( + isinstance(maybe_type_var, node_classes.Tuple) + and not maybe_type_var.elts + ): + # The typing object is subscriptable if the second argument of the _alias function # is a TypeVar or a tuple of TypeVar. We could check the type of the second argument but - # it appears that in the typing module the second argument is only TypeVar or a tuple of TypeVar or empty tuple. + #  it appears that in the typing module the second argument is only TypeVar or a tuple of TypeVar or empty tuple. # This last value means the type is not Generic and thus cannot be subscriptable func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) res.locals["__class_getitem__"] = [func_to_add] diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index fb802ba1fb..e17a4fee3c 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -2618,17 +2618,19 @@ def getitem(self, index, context=None): methods = dunder_lookup.lookup(self, "__getitem__") except exceptions.AttributeInferenceError as exc: if isinstance(self, ClassDef): - # subscripting a class definition may be - # achieved thanks to __class_getitem__ method - # which is a classmethod defined in the class - # that supports subscript and not in the metaclass + # subscripting a class definition may be + #  achieved thanks to __class_getitem__ method + #  which is a classmethod defined in the class + #  that supports subscript and not in the metaclass try: methods = self.getattr("__class_getitem__") # Here it is assumed that the __class_getitem__ node is - # a FunctionDef. One possible improvment would be to deal + #  a FunctionDef. One possible improvment would be to deal # with more generic inference. except exceptions.AttributeInferenceError: - raise exceptions.AstroidTypeError(node=self, context=context) from exc + raise exceptions.AstroidTypeError( + node=self, context=context + ) from exc else: raise exceptions.AstroidTypeError(node=self, context=context) from exc diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index bb56efa0a8..bd9d924634 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1066,7 +1066,7 @@ def test_collections_object_not_subscriptable(self): ], ) with self.assertRaises(astroid.exceptions.AttributeInferenceError): - inferred.getattr('__class_getitem__') + inferred.getattr("__class_getitem__") @test_utils.require_version(minver="3.9") def test_collections_object_subscriptable(self): @@ -1089,10 +1089,12 @@ def test_collections_object_subscriptable(self): "Sized", "Iterable", "Container", - "object" - ] + "object", + ], + ) + self.assertIsInstance( + inferred.getattr("__class_getitem__")[0], nodes.FunctionDef ) - self.assertIsInstance(inferred.getattr('__class_getitem__')[0], nodes.FunctionDef) @test_utils.require_version(maxver="3.9") def test_collections_object_not_yet_subscriptable(self): @@ -1127,11 +1129,11 @@ def test_collections_object_not_yet_subscriptable(self): "Sized", "Iterable", "Container", - "object" + "object", ], ) with self.assertRaises(astroid.exceptions.AttributeInferenceError): - inferred.getattr('__class_getitem__') + inferred.getattr("__class_getitem__") @test_utils.require_version(minver="3.9") def test_collections_object_subscriptable_2(self): @@ -1169,6 +1171,7 @@ def test_collections_object_not_yet_subscriptable_2(self): with self.assertRaises(astroid.exceptions.InferenceError): next(node.infer()) + @test_utils.require_version("3.6") class TypingBrain(unittest.TestCase): def test_namedtuple_base(self): @@ -1353,7 +1356,7 @@ class CustomTD(TypedDict): assert len(typing_module.locals["TypedDict"]) == 1 assert inferred_base == typing_module.locals["TypedDict"][0] - @test_utils.require_version(minver="3.7") + @test_utils.require_version(minver="3.7") def test_typing_alias_type(self): """ Test that the type aliased thanks to typing._alias function are @@ -1395,8 +1398,8 @@ class Derived2(typing.OrderedDict[int, str]): """ ) inferred = next(node.infer()) - # OrderedDict has no metaclass because it - # inherits from dict which is C coded + #  OrderedDict has no metaclass because it + #  inherits from dict which is C coded self.assertIsNone(inferred.metaclass()) assertEqualMro( inferred, @@ -1436,9 +1439,9 @@ def test_typing_object_not_subscriptable(self): ], ) with self.assertRaises(astroid.exceptions.AttributeInferenceError): - inferred.getattr('__class_getitem__') + inferred.getattr("__class_getitem__") - @test_utils.require_version(minver="3.7") + @test_utils.require_version(minver="3.7") def test_typing_object_subscriptable(self): """Test that MutableSet is subscriptable""" right_node = builder.extract_node( @@ -1459,10 +1462,13 @@ def test_typing_object_subscriptable(self): "Sized", "Iterable", "Container", - "object" - ] + "object", + ], + ) + self.assertIsInstance( + inferred.getattr("__class_getitem__")[0], nodes.FunctionDef ) - self.assertIsInstance(inferred.getattr('__class_getitem__')[0], nodes.FunctionDef) + class ReBrainTest(unittest.TestCase): def test_regex_flags(self): From 5d58132f60e57a20baf6d39abd043c62a2a21c6c Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 15:53:45 +0100 Subject: [PATCH 38/59] Adds two entries --- ChangeLog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index b8d318d59d..b7cc14d95c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,12 @@ What's New in astroid 2.5.2? ============================ Release Date: TBA +* Takes into account the fact that subscript inferring for a ClassDef may involve __class_getitem__ method + +* Reworks the `collections` and `typing` brain so that `pylint`s acceptance tests are fine. + + Closes PyCQA/pylint#4206 + * Detects `import numpy` as a valid `numpy` import. Closes PyCQA/pylint#3974 From fa7b6e7b517d48071a3fbe6d6c3709a00641aa34 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 18:20:53 +0100 Subject: [PATCH 39/59] Extends the filter to determine what is subscriptable to include OrderedDict --- astroid/brain/brain_collections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 5ac0cf73a9..40e0da78e1 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -110,7 +110,7 @@ def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: :param node: ClassDef node """ - if node.qname().startswith("_collections_abc"): + if node.qname().startswith("_collections") or node.qname().startswith("collections"): try: node.getattr("__class_getitem__") return True From d8759f3ad4168a089c9c32e4c1eb4ea3426a32e8 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 14 Mar 2021 18:27:02 +0100 Subject: [PATCH 40/59] Formatting according to black --- astroid/brain/brain_collections.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 40e0da78e1..3c8d85f82b 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -110,7 +110,9 @@ def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: :param node: ClassDef node """ - if node.qname().startswith("_collections") or node.qname().startswith("collections"): + if node.qname().startswith("_collections") or node.qname().startswith( + "collections" + ): try: node.getattr("__class_getitem__") return True From d8ee527974b7e969348b275927b9c15848e692d4 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 21 Mar 2021 11:44:16 +0100 Subject: [PATCH 41/59] Takes into account @AWhetter remarks --- astroid/brain/brain_collections.py | 8 +++----- astroid/scoped_nodes.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 3c8d85f82b..5459e6e169 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -88,7 +88,8 @@ def _collections_module_properties(node, context=None): """ Adds a path to a fictive file as the _collections module is a pure C lib. """ - node.file = "/tmp/unknown" + collections_mod = node.import_module('collections') + node.file = collections_mod.file return node @@ -100,9 +101,6 @@ def _collections_module_properties(node, context=None): ) -PY39 = sys.version_info >= (3, 9) - - def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: """ Returns True if the node corresponds to a ClassDef of the Collections.abc module that @@ -130,7 +128,7 @@ def __class_getitem__(cls, item): def easy_class_getitem_inference(node, context=None): #  Here __class_getitem__ exists but is quite a mess to infer thus - #  put instead an easy inference tip + #  put an easy inference tip func_to_add = astroid.extract_node(CLASS_GET_ITEM_TEMPLATE) node.locals["__class_getitem__"] = [func_to_add] diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index e17a4fee3c..4d62cb3636 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -2625,7 +2625,7 @@ def getitem(self, index, context=None): try: methods = self.getattr("__class_getitem__") # Here it is assumed that the __class_getitem__ node is - #  a FunctionDef. One possible improvment would be to deal + #  a FunctionDef. One possible improvement would be to deal # with more generic inference. except exceptions.AttributeInferenceError: raise exceptions.AstroidTypeError( From 8ff28937a56994fd3f7ee573d024b78acf306b34 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 21 Mar 2021 19:32:37 +0100 Subject: [PATCH 42/59] Deactivates access to __class_getitem__ method --- astroid/brain/brain_typing.py | 38 +++++++++++++++++++++++++++++++++++ tests/unittest_brain.py | 32 +++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index af4b7e8b18..893040396e 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -19,6 +19,7 @@ nodes, context, InferenceError, + AttributeInferenceError ) import astroid @@ -140,6 +141,28 @@ def _looks_like_typing_alias(node: nodes.Call) -> bool: ) +def _forbid_class_getitem_access(node: nodes.ClassDef) -> None: + """ + Disable the access to __class_getitem__ method for the node in parameters + """ + if not isinstance(node, nodes.ClassDef): + raise TypeError("The parameter type should be ClassDef") + try: + node.getattr('__class_getitem__') + # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the + # protocol defined in collections module) whereas the typing module consider it should not + origin_func : node.getattr + # We do not want __class_getitem__ to be found in the classdef + def raiser(attr): + if attr == '__class_getitem__': + raise AttributeInferenceError("__class_getitem__ access is not allowed") + else: + return origin_func(attr) + node.getattr = raiser + except AttributeInferenceError: + pass + + def infer_typing_alias( node: nodes.Call, ctx: context.InferenceContext = None ) -> typing.Optional[node_classes.NodeNG]: @@ -176,6 +199,21 @@ def infer_typing_alias( # This last value means the type is not Generic and thus cannot be subscriptable func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) res.locals["__class_getitem__"] = [func_to_add] + elif isinstance(maybe_type_var, node_classes.Tuple) and not maybe_type_var.elts: + # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the + # protocol defined in collections module) whereas the typing module consider it should not + # We do not want __class_getitem__ to be found in the classdef + _forbid_class_getitem_access(res) + else: + # Within python3.9 discrepencies exist between some collections.abc containers that are subscriptable whereas + # corresponding containers in the typing module are not! This is the case at least for ByteString. + # It is far more to complex and dangerous to try to remove __class_getitem__ method from all the ancestors of the + # current class. Instead we raise an AttributeInferenceError if we try to access it. + maybe_type_var = node.args[1] + if isinstance(maybe_type_var, nodes.Const) and maybe_type_var.value == 0: + # Starting with Python39 the _alias function is in fact instantiation of _SpecialGenericAlias class. + # Thus the type is not Generic if the second argument of the call is equal to zero + _forbid_class_getitem_access(res) return res return None diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index bd9d924634..f5f224fd33 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1171,6 +1171,22 @@ def test_collections_object_not_yet_subscriptable_2(self): with self.assertRaises(astroid.exceptions.InferenceError): next(node.infer()) + @test_utils.require_version(minver="3.9") + def test_collections_object_subscriptable_3(self): + """With python39 ByteString class of the colletions module is subscritable (but not the same class from typing module)""" + right_node = builder.extract_node( + """ + import collections.abc + + collections.abc.ByteString[int] + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + self.assertIsInstance( + inferred.getattr("__class_getitem__")[0], nodes.FunctionDef + ) + @test_utils.require_version("3.6") class TypingBrain(unittest.TestCase): @@ -1469,6 +1485,22 @@ def test_typing_object_subscriptable(self): inferred.getattr("__class_getitem__")[0], nodes.FunctionDef ) + def test_typing_object_notsubscriptable_3(self): + """Until python39 ByteString class of the typing module is not subscritable (whereas it is in the collections module)""" + right_node = builder.extract_node( + """ + import typing + + typing.ByteString + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + self.assertIsInstance( + inferred.getattr("__class_getitem__")[0], nodes.FunctionDef + ) + class ReBrainTest(unittest.TestCase): def test_regex_flags(self): From 0f875c82749c0b75ac620027bbb8a8b84cff624c Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 27 Mar 2021 11:26:39 +0100 Subject: [PATCH 43/59] OrderedDict appears in typing module with python3.7.2 --- tests/unittest_brain.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index f5f224fd33..af7e3c7eb2 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1406,6 +1406,14 @@ class Derived1(MutableSet[T]): ], ) + @test_utils.require_version(minver="3.7.2") + def test_typing_alias_type_2(self): + """ + Test that the type aliased thanks to typing._alias function are + correctly inferred. + typing_alias function is introduced with python37. + OrderedDict in the typing module appears only with python 3.7.2 + """ node = builder.extract_node( """ import typing From edf1503f301bf1d3cf1fc40b4037e5f4ff13aa32 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 27 Mar 2021 11:27:14 +0100 Subject: [PATCH 44/59] _alias function in the typing module appears with python3.7 --- tests/unittest_brain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index af7e3c7eb2..8ee27edc0d 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1493,6 +1493,7 @@ def test_typing_object_subscriptable(self): inferred.getattr("__class_getitem__")[0], nodes.FunctionDef ) + @test_utils.require_version(minver="3.7") def test_typing_object_notsubscriptable_3(self): """Until python39 ByteString class of the typing module is not subscritable (whereas it is in the collections module)""" right_node = builder.extract_node( From 3357204fefac7084c8bb9876269f9d400cc85604 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 27 Mar 2021 11:28:52 +0100 Subject: [PATCH 45/59] Formatting according to black --- astroid/brain/brain_collections.py | 2 +- astroid/brain/brain_typing.py | 34 +++++++++++++++++------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 5459e6e169..47f892d0b6 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -88,7 +88,7 @@ def _collections_module_properties(node, context=None): """ Adds a path to a fictive file as the _collections module is a pure C lib. """ - collections_mod = node.import_module('collections') + collections_mod = node.import_module("collections") node.file = collections_mod.file return node diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 893040396e..d585301a34 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -19,7 +19,7 @@ nodes, context, InferenceError, - AttributeInferenceError + AttributeInferenceError, ) import astroid @@ -148,16 +148,17 @@ def _forbid_class_getitem_access(node: nodes.ClassDef) -> None: if not isinstance(node, nodes.ClassDef): raise TypeError("The parameter type should be ClassDef") try: - node.getattr('__class_getitem__') - # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the + node.getattr("__class_getitem__") + # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the # protocol defined in collections module) whereas the typing module consider it should not - origin_func : node.getattr - # We do not want __class_getitem__ to be found in the classdef + origin_func: node.getattr + #  We do not want __class_getitem__ to be found in the classdef def raiser(attr): - if attr == '__class_getitem__': + if attr == "__class_getitem__": raise AttributeInferenceError("__class_getitem__ access is not allowed") else: return origin_func(attr) + node.getattr = raiser except AttributeInferenceError: pass @@ -199,20 +200,23 @@ def infer_typing_alias( # This last value means the type is not Generic and thus cannot be subscriptable func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) res.locals["__class_getitem__"] = [func_to_add] - elif isinstance(maybe_type_var, node_classes.Tuple) and not maybe_type_var.elts: - # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the + elif ( + isinstance(maybe_type_var, node_classes.Tuple) + and not maybe_type_var.elts + ): + # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the # protocol defined in collections module) whereas the typing module consider it should not - # We do not want __class_getitem__ to be found in the classdef + #  We do not want __class_getitem__ to be found in the classdef _forbid_class_getitem_access(res) else: - # Within python3.9 discrepencies exist between some collections.abc containers that are subscriptable whereas - # corresponding containers in the typing module are not! This is the case at least for ByteString. - # It is far more to complex and dangerous to try to remove __class_getitem__ method from all the ancestors of the - # current class. Instead we raise an AttributeInferenceError if we try to access it. + #  Within python3.9 discrepencies exist between some collections.abc containers that are subscriptable whereas + #  corresponding containers in the typing module are not! This is the case at least for ByteString. + #  It is far more to complex and dangerous to try to remove __class_getitem__ method from all the ancestors of the + #  current class. Instead we raise an AttributeInferenceError if we try to access it. maybe_type_var = node.args[1] if isinstance(maybe_type_var, nodes.Const) and maybe_type_var.value == 0: - # Starting with Python39 the _alias function is in fact instantiation of _SpecialGenericAlias class. - # Thus the type is not Generic if the second argument of the call is equal to zero + #  Starting with Python39 the _alias function is in fact instantiation of _SpecialGenericAlias class. + #  Thus the type is not Generic if the second argument of the call is equal to zero _forbid_class_getitem_access(res) return res return None From 38cf33cb7cc67d0f3a436559252377edc494a112 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sat, 27 Mar 2021 19:01:41 +0100 Subject: [PATCH 46/59] _alias function is used also for builtins type and not only for collections.abc ones --- astroid/brain/brain_typing.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index d585301a34..7e176f3dcc 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -137,7 +137,12 @@ def _looks_like_typing_alias(node: nodes.Call) -> bool: isinstance(node, nodes.Call) and isinstance(node.func, nodes.Name) and node.func.name == "_alias" - and isinstance(node.args[0], nodes.Attribute) + and ( + # _alias function works also for builtins object such as list and dict + isinstance(node.args[0], nodes.Attribute) + or isinstance(node.args[0], nodes.Name) + and node.args[0].name not in ('type',) + ) ) @@ -218,6 +223,11 @@ def infer_typing_alias( #  Starting with Python39 the _alias function is in fact instantiation of _SpecialGenericAlias class. #  Thus the type is not Generic if the second argument of the call is equal to zero _forbid_class_getitem_access(res) + elif '__class_getitem__' in res.locals and isinstance(res.locals['__class_getitem__'][0], nodes.EmptyNode): + #  if res is a builtin object such as list, or dict then it has a __class_getitem__ function which is + #  not bound to the regular class instance but to an EmptyNode. Then we replace it. + func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) + res.locals["__class_getitem__"] = [func_to_add] return res return None From 15bdc781287f44c3fd0c39a94f3053889f8b3c67 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Mon, 29 Mar 2021 16:27:38 +0200 Subject: [PATCH 47/59] Adds tests for both builtins type that are subscriptable and typing builtin alias type that are also subscriptable --- tests/unittest_brain.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 8ee27edc0d..1b4eb19d34 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -32,6 +32,7 @@ # For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER """Tests for basic functionality in astroid.brain.""" +from astroid.scoped_nodes import ClassDef, FunctionDef import io import queue import re @@ -1026,6 +1027,20 @@ def test_invalid_type_subscript(self): with self.assertRaises(astroid.exceptions.AttributeInferenceError): meth_inf = val_inf.getattr("__class_getitem__")[0] + @test_utils.require_version(minver="3.9") + def test_builtin_subscriptable(self): + """ + Starting with python3.9 builtin type such as list are subscriptable + """ + for typename in ("tuple", "list", "dict", "set", "frozenset"): + src = """ + {:s}[int] + """.format(typename) + right_node = builder.extract_node(src) + inferred = next(right_node.infer()) + self.assertIsInstance(inferred, ClassDef) + self.assertIsInstance(inferred.getattr('__iter__')[0], FunctionDef) + def check_metaclass_is_abc(node: nodes.ClassDef): meta = node.metaclass() @@ -1510,6 +1525,22 @@ def test_typing_object_notsubscriptable_3(self): inferred.getattr("__class_getitem__")[0], nodes.FunctionDef ) + @test_utils.require_version(minver="3.9") + def test_typing_object_builtin_subscriptable(self): + """ + Test that builtins alias, such as typing.List, are subscriptable + """ + # Do not test Tuple as it is inferred as _TupleType class (needs a brain?) + for typename in ("List", "Dict", "Set", "FrozenSet"): + src = """ + import typing + typing.{:s}[int] + """.format(typename) + right_node = builder.extract_node(src) + inferred = next(right_node.infer()) + self.assertIsInstance(inferred, ClassDef) + self.assertIsInstance(inferred.getattr('__iter__')[0], FunctionDef) + class ReBrainTest(unittest.TestCase): def test_regex_flags(self): From c5d1c40b8181395d83e757bf7b5b33a57a5bba41 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Mon, 29 Mar 2021 16:28:18 +0200 Subject: [PATCH 48/59] No need to handle builtin types in this brain. It is better suited inside brain_bulitin_inference --- astroid/brain/brain_typing.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 7e176f3dcc..49846cab4d 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -223,11 +223,6 @@ def infer_typing_alias( #  Starting with Python39 the _alias function is in fact instantiation of _SpecialGenericAlias class. #  Thus the type is not Generic if the second argument of the call is equal to zero _forbid_class_getitem_access(res) - elif '__class_getitem__' in res.locals and isinstance(res.locals['__class_getitem__'][0], nodes.EmptyNode): - #  if res is a builtin object such as list, or dict then it has a __class_getitem__ function which is - #  not bound to the regular class instance but to an EmptyNode. Then we replace it. - func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) - res.locals["__class_getitem__"] = [func_to_add] return res return None From ede0e009c2cb9f564799848c875fbb320abc2397 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Mon, 29 Mar 2021 16:29:11 +0200 Subject: [PATCH 49/59] Adds brain to handle builtin types that are subscriptable starting with python39 --- astroid/brain/brain_builtin_inference.py | 51 +++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/astroid/brain/brain_builtin_inference.py b/astroid/brain/brain_builtin_inference.py index 2e0cab27be..637b04d2b3 100644 --- a/astroid/brain/brain_builtin_inference.py +++ b/astroid/brain/brain_builtin_inference.py @@ -19,12 +19,13 @@ """Astroid hooks for various builtins.""" from functools import partial -from textwrap import dedent +import sys from astroid import ( MANAGER, UseInferenceDefault, AttributeInferenceError, + extract_node, inference_tip, InferenceError, NameInferenceError, @@ -40,6 +41,9 @@ from astroid import util +PY39 = sys.version_info[:2] >= (3, 9) + + OBJECT_DUNDER_NEW = "object.__new__" STR_CLASS = """ @@ -882,6 +886,46 @@ def _build_dict_with_elements(elements): return _build_dict_with_elements([]) +def _looks_like_subscriptable_types(node): + """ + Try to figure out if a Name node corresponds to a subscriptable builtin type + + :param node: node to check + :type node: astroid.node_classes.NodeNG + :return: true if the node is a Name node corresponding to a subscriptable builtin type + :rtype: bool + """ + if isinstance(node, nodes.Name) and node.name in ("tuple", "list", "dict", "set", "frozenset"): + return True + return False + + +CLASS_GETITEM_TEMPLATE = """ +@classmethod +def __class_getitem__(cls, item): + return cls +""" + + +def replace_class_getitem(node, context=None): + """ + Starting with python39 some builtins types are subscriptalbe. + However the __class_getitem__ method is attached to an EmptyNode + which prevents any correct inference of subscript by the mean of + ClassDef.getitem method. Thus we replace this method with an inferable + one + """ + cls_node = next(node.infer()) + if cls_node is util.Uninferable: + return cls_node + if not '__class_getitem__' in cls_node.locals: + return cls_node + if isinstance(cls_node.locals['__class_getitem__'][0], nodes.EmptyNode): + func_to_add = extract_node(CLASS_GETITEM_TEMPLATE) + cls_node.locals["__class_getitem__"] = [func_to_add] + return cls_node + + # Builtins inference register_builtin_transform(infer_bool, "bool") register_builtin_transform(infer_super, "super") @@ -910,3 +954,8 @@ def _build_dict_with_elements(elements): inference_tip(_infer_object__new__decorator), _infer_object__new__decorator_check, ) + +if PY39: + MANAGER.register_transform( + nodes.Name, replace_class_getitem, _looks_like_subscriptable_types + ) \ No newline at end of file From a09f8386f2820c5a511c3333cbde0d8f2d1a05b2 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Mon, 29 Mar 2021 16:33:07 +0200 Subject: [PATCH 50/59] Formatting according to black --- astroid/brain/brain_builtin_inference.py | 14 ++++++++++---- astroid/brain/brain_typing.py | 4 ++-- tests/unittest_brain.py | 14 +++++++++----- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/astroid/brain/brain_builtin_inference.py b/astroid/brain/brain_builtin_inference.py index 637b04d2b3..560de3a7e1 100644 --- a/astroid/brain/brain_builtin_inference.py +++ b/astroid/brain/brain_builtin_inference.py @@ -895,7 +895,13 @@ def _looks_like_subscriptable_types(node): :return: true if the node is a Name node corresponding to a subscriptable builtin type :rtype: bool """ - if isinstance(node, nodes.Name) and node.name in ("tuple", "list", "dict", "set", "frozenset"): + if isinstance(node, nodes.Name) and node.name in ( + "tuple", + "list", + "dict", + "set", + "frozenset", + ): return True return False @@ -918,9 +924,9 @@ def replace_class_getitem(node, context=None): cls_node = next(node.infer()) if cls_node is util.Uninferable: return cls_node - if not '__class_getitem__' in cls_node.locals: + if not "__class_getitem__" in cls_node.locals: return cls_node - if isinstance(cls_node.locals['__class_getitem__'][0], nodes.EmptyNode): + if isinstance(cls_node.locals["__class_getitem__"][0], nodes.EmptyNode): func_to_add = extract_node(CLASS_GETITEM_TEMPLATE) cls_node.locals["__class_getitem__"] = [func_to_add] return cls_node @@ -958,4 +964,4 @@ def replace_class_getitem(node, context=None): if PY39: MANAGER.register_transform( nodes.Name, replace_class_getitem, _looks_like_subscriptable_types - ) \ No newline at end of file + ) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 49846cab4d..e64d790e93 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -138,10 +138,10 @@ def _looks_like_typing_alias(node: nodes.Call) -> bool: and isinstance(node.func, nodes.Name) and node.func.name == "_alias" and ( - # _alias function works also for builtins object such as list and dict + #  _alias function works also for builtins object such as list and dict isinstance(node.args[0], nodes.Attribute) or isinstance(node.args[0], nodes.Name) - and node.args[0].name not in ('type',) + and node.args[0].name not in ("type",) ) ) diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 1b4eb19d34..eea5295cb6 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1035,11 +1035,13 @@ def test_builtin_subscriptable(self): for typename in ("tuple", "list", "dict", "set", "frozenset"): src = """ {:s}[int] - """.format(typename) + """.format( + typename + ) right_node = builder.extract_node(src) inferred = next(right_node.infer()) self.assertIsInstance(inferred, ClassDef) - self.assertIsInstance(inferred.getattr('__iter__')[0], FunctionDef) + self.assertIsInstance(inferred.getattr("__iter__")[0], FunctionDef) def check_metaclass_is_abc(node: nodes.ClassDef): @@ -1530,16 +1532,18 @@ def test_typing_object_builtin_subscriptable(self): """ Test that builtins alias, such as typing.List, are subscriptable """ - # Do not test Tuple as it is inferred as _TupleType class (needs a brain?) + #  Do not test Tuple as it is inferred as _TupleType class (needs a brain?) for typename in ("List", "Dict", "Set", "FrozenSet"): src = """ import typing typing.{:s}[int] - """.format(typename) + """.format( + typename + ) right_node = builder.extract_node(src) inferred = next(right_node.infer()) self.assertIsInstance(inferred, ClassDef) - self.assertIsInstance(inferred.getattr('__iter__')[0], FunctionDef) + self.assertIsInstance(inferred.getattr("__iter__")[0], FunctionDef) class ReBrainTest(unittest.TestCase): From d71a7aad74916ecf4a8f7c444340cf9b377f71dc Mon Sep 17 00:00:00 2001 From: hippo91 Date: Wed, 31 Mar 2021 15:29:45 +0200 Subject: [PATCH 51/59] Uses partial function instead of closure in order pylint acceptance to be ok --- astroid/brain/brain_typing.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index e64d790e93..9c4f59c68a 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -8,7 +8,7 @@ """Astroid hooks for typing.py support.""" import sys import typing -from functools import lru_cache +from functools import lru_cache, partial from astroid import ( MANAGER, @@ -146,6 +146,18 @@ def _looks_like_typing_alias(node: nodes.Call) -> bool: ) +def full_raiser(origin_func, attr, *args, **kwargs): + """ + Raises an AttributeInferenceError in case of access to __class_getitem__ method. + Otherwise just call origin_func. + """ + if attr == "__class_getitem__": + raise AttributeInferenceError("__class_getitem__ access is not allowed") + else: + return origin_func(attr, *args, **kwargs) + + + def _forbid_class_getitem_access(node: nodes.ClassDef) -> None: """ Disable the access to __class_getitem__ method for the node in parameters @@ -156,15 +168,9 @@ def _forbid_class_getitem_access(node: nodes.ClassDef) -> None: node.getattr("__class_getitem__") # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the # protocol defined in collections module) whereas the typing module consider it should not - origin_func: node.getattr #  We do not want __class_getitem__ to be found in the classdef - def raiser(attr): - if attr == "__class_getitem__": - raise AttributeInferenceError("__class_getitem__ access is not allowed") - else: - return origin_func(attr) - - node.getattr = raiser + partial_raiser = partial(full_raiser, node.getattr) + node.getattr = partial_raiser except AttributeInferenceError: pass From 1be6d3fe6b327ad0c5d34c43c0407fe577284a7d Mon Sep 17 00:00:00 2001 From: hippo91 Date: Wed, 31 Mar 2021 15:31:38 +0200 Subject: [PATCH 52/59] Handling the __class_getitem__ method associated to EmptyNode for builtin types is made directly inside the getitem method --- astroid/brain/brain_builtin_inference.py | 30 ------------------------ astroid/scoped_nodes.py | 6 +++++ 2 files changed, 6 insertions(+), 30 deletions(-) diff --git a/astroid/brain/brain_builtin_inference.py b/astroid/brain/brain_builtin_inference.py index 560de3a7e1..b5d19cff93 100644 --- a/astroid/brain/brain_builtin_inference.py +++ b/astroid/brain/brain_builtin_inference.py @@ -906,31 +906,6 @@ def _looks_like_subscriptable_types(node): return False -CLASS_GETITEM_TEMPLATE = """ -@classmethod -def __class_getitem__(cls, item): - return cls -""" - - -def replace_class_getitem(node, context=None): - """ - Starting with python39 some builtins types are subscriptalbe. - However the __class_getitem__ method is attached to an EmptyNode - which prevents any correct inference of subscript by the mean of - ClassDef.getitem method. Thus we replace this method with an inferable - one - """ - cls_node = next(node.infer()) - if cls_node is util.Uninferable: - return cls_node - if not "__class_getitem__" in cls_node.locals: - return cls_node - if isinstance(cls_node.locals["__class_getitem__"][0], nodes.EmptyNode): - func_to_add = extract_node(CLASS_GETITEM_TEMPLATE) - cls_node.locals["__class_getitem__"] = [func_to_add] - return cls_node - # Builtins inference register_builtin_transform(infer_bool, "bool") @@ -960,8 +935,3 @@ def replace_class_getitem(node, context=None): inference_tip(_infer_object__new__decorator), _infer_object__new__decorator_check, ) - -if PY39: - MANAGER.register_transform( - nodes.Name, replace_class_getitem, _looks_like_subscriptable_types - ) diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index 4d62cb3636..f935de4650 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -54,6 +54,8 @@ from astroid import util +PY39 = sys.version_info[:2] >= (3, 9) + BUILTINS = builtins.__name__ ITER_METHODS = ("__iter__", "__getitem__") EXCEPTION_BASE_CLASSES = frozenset({"Exception", "BaseException"}) @@ -2642,6 +2644,10 @@ def getitem(self, index, context=None): try: return next(method.infer_call_result(self, new_context)) + except AttributeError: + if isinstance(method, node_classes.EmptyNode) and self.name in ('list', 'dict', 'set', 'tuple', 'frozenset') and PY39: + return self + raise except exceptions.InferenceError: return util.Uninferable From aaa9543892a882e8ece98bcba4db830a20477897 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Wed, 31 Mar 2021 15:44:28 +0200 Subject: [PATCH 53/59] infer_typing_alias has to be an inference_tip to avoid interferences between typing module and others (collections or builtin) --- astroid/brain/brain_typing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 9c4f59c68a..b101c317c1 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -229,8 +229,8 @@ def infer_typing_alias( #  Starting with Python39 the _alias function is in fact instantiation of _SpecialGenericAlias class. #  Thus the type is not Generic if the second argument of the call is equal to zero _forbid_class_getitem_access(res) - return res - return None + return iter([res]) + return iter([astroid.Uninferable]) MANAGER.register_transform( @@ -248,4 +248,4 @@ def infer_typing_alias( ) if PY37: - MANAGER.register_transform(nodes.Call, infer_typing_alias, _looks_like_typing_alias) + MANAGER.register_transform(nodes.Call, inference_tip(infer_typing_alias), _looks_like_typing_alias) From 56ab1126815b052c8cf819594eb0bfe0149537e3 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Wed, 31 Mar 2021 15:46:50 +0200 Subject: [PATCH 54/59] Formatting --- astroid/brain/brain_builtin_inference.py | 1 - astroid/brain/brain_typing.py | 5 +++-- astroid/scoped_nodes.py | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/astroid/brain/brain_builtin_inference.py b/astroid/brain/brain_builtin_inference.py index b5d19cff93..24db431611 100644 --- a/astroid/brain/brain_builtin_inference.py +++ b/astroid/brain/brain_builtin_inference.py @@ -906,7 +906,6 @@ def _looks_like_subscriptable_types(node): return False - # Builtins inference register_builtin_transform(infer_bool, "bool") register_builtin_transform(infer_super, "super") diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index b101c317c1..09d5465fbf 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -157,7 +157,6 @@ def full_raiser(origin_func, attr, *args, **kwargs): return origin_func(attr, *args, **kwargs) - def _forbid_class_getitem_access(node: nodes.ClassDef) -> None: """ Disable the access to __class_getitem__ method for the node in parameters @@ -248,4 +247,6 @@ def infer_typing_alias( ) if PY37: - MANAGER.register_transform(nodes.Call, inference_tip(infer_typing_alias), _looks_like_typing_alias) + MANAGER.register_transform( + nodes.Call, inference_tip(infer_typing_alias), _looks_like_typing_alias + ) diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index f935de4650..665b09722c 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -2645,7 +2645,11 @@ def getitem(self, index, context=None): try: return next(method.infer_call_result(self, new_context)) except AttributeError: - if isinstance(method, node_classes.EmptyNode) and self.name in ('list', 'dict', 'set', 'tuple', 'frozenset') and PY39: + if ( + isinstance(method, node_classes.EmptyNode) + and self.name in ("list", "dict", "set", "tuple", "frozenset") + and PY39 + ): return self raise except exceptions.InferenceError: From 80b8ca952f4066f1f89746e537ef169966c2c18e Mon Sep 17 00:00:00 2001 From: hippo91 Date: Wed, 31 Mar 2021 16:40:48 +0200 Subject: [PATCH 55/59] Removes useless code --- astroid/brain/brain_builtin_inference.py | 25 ------------------------ 1 file changed, 25 deletions(-) diff --git a/astroid/brain/brain_builtin_inference.py b/astroid/brain/brain_builtin_inference.py index 24db431611..ef63d17eea 100644 --- a/astroid/brain/brain_builtin_inference.py +++ b/astroid/brain/brain_builtin_inference.py @@ -19,13 +19,11 @@ """Astroid hooks for various builtins.""" from functools import partial -import sys from astroid import ( MANAGER, UseInferenceDefault, AttributeInferenceError, - extract_node, inference_tip, InferenceError, NameInferenceError, @@ -41,9 +39,6 @@ from astroid import util -PY39 = sys.version_info[:2] >= (3, 9) - - OBJECT_DUNDER_NEW = "object.__new__" STR_CLASS = """ @@ -886,26 +881,6 @@ def _build_dict_with_elements(elements): return _build_dict_with_elements([]) -def _looks_like_subscriptable_types(node): - """ - Try to figure out if a Name node corresponds to a subscriptable builtin type - - :param node: node to check - :type node: astroid.node_classes.NodeNG - :return: true if the node is a Name node corresponding to a subscriptable builtin type - :rtype: bool - """ - if isinstance(node, nodes.Name) and node.name in ( - "tuple", - "list", - "dict", - "set", - "frozenset", - ): - return True - return False - - # Builtins inference register_builtin_transform(infer_bool, "bool") register_builtin_transform(infer_super, "super") From 30b1e795c057b8688d1251bd0477ca1fec03dbed Mon Sep 17 00:00:00 2001 From: hippo91 Date: Wed, 31 Mar 2021 16:44:13 +0200 Subject: [PATCH 56/59] Adds comment --- astroid/scoped_nodes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index c9c210a378..3c2b73370b 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -2645,6 +2645,11 @@ def getitem(self, index, context=None): try: return next(method.infer_call_result(self, new_context)) except AttributeError: + # Starting with python3.9, builtin types list, dict etc... + # are subscriptable thanks to __class_getitem___ classmethod. + #  However in such case the method is bound to an EmptyNode and + #  EmptyNode doesn't have infer_call_result method yielding to + #  AttributeError if ( isinstance(method, node_classes.EmptyNode) and self.name in ("list", "dict", "set", "tuple", "frozenset") From 0bc7d6be724f680b806eb7b8f3fa52a4405969e9 Mon Sep 17 00:00:00 2001 From: hippo91 Date: Fri, 2 Apr 2021 11:48:51 +0200 Subject: [PATCH 57/59] Takes into account @cdce8p remarks --- astroid/brain/brain_collections.py | 14 +------------- astroid/brain/brain_typing.py | 30 ++++++++++++------------------ tests/unittest_brain.py | 9 ++++----- 3 files changed, 17 insertions(+), 36 deletions(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 47f892d0b6..64e7ea82a0 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -84,20 +84,8 @@ def __class_getitem__(cls, item): return cls""" return base_ordered_dict_class -def _collections_module_properties(node, context=None): - """ - Adds a path to a fictive file as the _collections module is a pure C lib. - """ - collections_mod = node.import_module("collections") - node.file = collections_mod.file - return node - - -astroid.MANAGER.register_transform( - astroid.Module, _collections_module_properties, lambda n: n.name == "_collections" -) astroid.register_module_extender( - astroid.MANAGER, "_collections", _collections_transform + astroid.MANAGER, "collections", _collections_transform ) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 11e9ec1064..cb29dc812a 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -141,26 +141,25 @@ def _looks_like_typing_alias(node: nodes.Call) -> bool: #  _alias function works also for builtins object such as list and dict isinstance(node.args[0], nodes.Attribute) or isinstance(node.args[0], nodes.Name) - and node.args[0].name not in ("type",) + and node.args[0].name != "type" ) ) -def full_raiser(origin_func, attr, *args, **kwargs): - """ - Raises an AttributeInferenceError in case of access to __class_getitem__ method. - Otherwise just call origin_func. - """ - if attr == "__class_getitem__": - raise AttributeInferenceError("__class_getitem__ access is not allowed") - else: - return origin_func(attr, *args, **kwargs) - - def _forbid_class_getitem_access(node: nodes.ClassDef) -> None: """ Disable the access to __class_getitem__ method for the node in parameters """ + def full_raiser(origin_func, attr, *args, **kwargs): + """ + Raises an AttributeInferenceError in case of access to __class_getitem__ method. + Otherwise just call origin_func. + """ + if attr == "__class_getitem__": + raise AttributeInferenceError("__class_getitem__ access is not allowed") + else: + return origin_func(attr, *args, **kwargs) + if not isinstance(node, nodes.ClassDef): raise TypeError("The parameter type should be ClassDef") try: @@ -183,8 +182,6 @@ def infer_typing_alias( :param node: call node :param context: inference context """ - if not isinstance(node, nodes.Call): - return None res = next(node.args[0].infer(context=ctx)) if res != astroid.Uninferable and isinstance(res, nodes.ClassDef): @@ -210,10 +207,7 @@ def infer_typing_alias( # This last value means the type is not Generic and thus cannot be subscriptable func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) res.locals["__class_getitem__"] = [func_to_add] - elif ( - isinstance(maybe_type_var, node_classes.Tuple) - and not maybe_type_var.elts - ): + else: # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the # protocol defined in collections module) whereas the typing module consider it should not #  We do not want __class_getitem__ to be found in the classdef diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index fb0590954b..d80dee6699 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -32,7 +32,6 @@ # For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER """Tests for basic functionality in astroid.brain.""" -from astroid.scoped_nodes import ClassDef, FunctionDef import io import queue import re @@ -1040,8 +1039,8 @@ def test_builtin_subscriptable(self): ) right_node = builder.extract_node(src) inferred = next(right_node.infer()) - self.assertIsInstance(inferred, ClassDef) - self.assertIsInstance(inferred.getattr("__iter__")[0], FunctionDef) + self.assertIsInstance(inferred, nodes.ClassDef) + self.assertIsInstance(inferred.getattr("__iter__")[0], nodes.FunctionDef) def check_metaclass_is_abc(node: nodes.ClassDef): @@ -1542,8 +1541,8 @@ def test_typing_object_builtin_subscriptable(self): ) right_node = builder.extract_node(src) inferred = next(right_node.infer()) - self.assertIsInstance(inferred, ClassDef) - self.assertIsInstance(inferred.getattr("__iter__")[0], FunctionDef) + self.assertIsInstance(inferred, nodes.ClassDef) + self.assertIsInstance(inferred.getattr("__iter__")[0], nodes.FunctionDef) class ReBrainTest(unittest.TestCase): From 5cd0fcbc61deb30c742cc6ea2afe3ffab3fa0c3a Mon Sep 17 00:00:00 2001 From: hippo91 Date: Sun, 4 Apr 2021 11:09:21 +0200 Subject: [PATCH 58/59] Formatting --- astroid/brain/brain_collections.py | 4 +--- astroid/brain/brain_typing.py | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 64e7ea82a0..91acf1b3cf 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -84,9 +84,7 @@ def __class_getitem__(cls, item): return cls""" return base_ordered_dict_class -astroid.register_module_extender( - astroid.MANAGER, "collections", _collections_transform -) +astroid.register_module_extender(astroid.MANAGER, "collections", _collections_transform) def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index cb29dc812a..86ecf24fc5 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -150,6 +150,7 @@ def _forbid_class_getitem_access(node: nodes.ClassDef) -> None: """ Disable the access to __class_getitem__ method for the node in parameters """ + def full_raiser(origin_func, attr, *args, **kwargs): """ Raises an AttributeInferenceError in case of access to __class_getitem__ method. From 2d5614165e1b94eaa67316067491789b4c03360c Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 6 Apr 2021 11:13:32 +0200 Subject: [PATCH 59/59] Style changes --- astroid/brain/brain_builtin_inference.py | 1 + astroid/brain/brain_collections.py | 12 ++++----- astroid/brain/brain_typing.py | 34 ++++++++++++------------ astroid/scoped_nodes.py | 14 +++++----- tests/unittest_brain.py | 21 +++------------ 5 files changed, 35 insertions(+), 47 deletions(-) diff --git a/astroid/brain/brain_builtin_inference.py b/astroid/brain/brain_builtin_inference.py index ef63d17eea..2e0cab27be 100644 --- a/astroid/brain/brain_builtin_inference.py +++ b/astroid/brain/brain_builtin_inference.py @@ -19,6 +19,7 @@ """Astroid hooks for various builtins.""" from functools import partial +from textwrap import dedent from astroid import ( MANAGER, diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 91acf1b3cf..031325ea6f 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -100,7 +100,7 @@ def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: try: node.getattr("__class_getitem__") return True - except: + except astroid.AttributeInferenceError: pass return False @@ -113,16 +113,16 @@ def __class_getitem__(cls, item): def easy_class_getitem_inference(node, context=None): - #  Here __class_getitem__ exists but is quite a mess to infer thus - #  put an easy inference tip + # Here __class_getitem__ exists but is quite a mess to infer thus + # put an easy inference tip func_to_add = astroid.extract_node(CLASS_GET_ITEM_TEMPLATE) node.locals["__class_getitem__"] = [func_to_add] if PY39: - #  Starting with Python39 some objects of the collection module are subscriptable - #  thanks to the __class_getitem__ method but the way it is implemented in - #  _collection_abc makes it difficult to infer. (We would have to handle AssignName inference in the + # Starting with Python39 some objects of the collection module are subscriptable + # thanks to the __class_getitem__ method but the way it is implemented in + # _collection_abc makes it difficult to infer. (We would have to handle AssignName inference in the # getitem method of the ClassDef class) Instead we put here a mock of the __class_getitem__ method astroid.MANAGER.register_transform( astroid.nodes.ClassDef, easy_class_getitem_inference, _looks_like_subscriptable diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 86ecf24fc5..5fa3c9edfb 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -8,7 +8,7 @@ """Astroid hooks for typing.py support.""" import sys import typing -from functools import lru_cache, partial +from functools import partial from astroid import ( MANAGER, @@ -138,7 +138,7 @@ def _looks_like_typing_alias(node: nodes.Call) -> bool: and isinstance(node.func, nodes.Name) and node.func.name == "_alias" and ( - #  _alias function works also for builtins object such as list and dict + # _alias function works also for builtins object such as list and dict isinstance(node.args[0], nodes.Attribute) or isinstance(node.args[0], nodes.Name) and node.args[0].name != "type" @@ -167,7 +167,7 @@ def full_raiser(origin_func, attr, *args, **kwargs): node.getattr("__class_getitem__") # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the # protocol defined in collections module) whereas the typing module consider it should not - #  We do not want __class_getitem__ to be found in the classdef + # We do not want __class_getitem__ to be found in the classdef partial_raiser = partial(full_raiser, node.getattr) node.getattr = partial_raiser except AttributeInferenceError: @@ -187,16 +187,16 @@ def infer_typing_alias( if res != astroid.Uninferable and isinstance(res, nodes.ClassDef): if not PY39: - #  Here the node is a typing object which is an alias toward + # Here the node is a typing object which is an alias toward # the corresponding object of collection.abc module. # Before python3.9 there is no subscript allowed for any of the collections.abc objects. # The subscript ability is given through the typing._GenericAlias class - #  which is the metaclass of the typing object but not the metaclass of the inferred - #  collections.abc object. + # which is the metaclass of the typing object but not the metaclass of the inferred + # collections.abc object. # Thus we fake subscript ability of the collections.abc object - #  by mocking the existence of a __class_getitem__ method. - #  We can not add `__getitem__` method in the metaclass of the object because - #  the metaclass is shared by subscriptable and not subscriptable object + # by mocking the existence of a __class_getitem__ method. + # We can not add `__getitem__` method in the metaclass of the object because + # the metaclass is shared by subscriptable and not subscriptable object maybe_type_var = node.args[1] if not ( isinstance(maybe_type_var, node_classes.Tuple) @@ -204,24 +204,24 @@ def infer_typing_alias( ): # The typing object is subscriptable if the second argument of the _alias function # is a TypeVar or a tuple of TypeVar. We could check the type of the second argument but - #  it appears that in the typing module the second argument is only TypeVar or a tuple of TypeVar or empty tuple. + # it appears that in the typing module the second argument is only TypeVar or a tuple of TypeVar or empty tuple. # This last value means the type is not Generic and thus cannot be subscriptable func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) res.locals["__class_getitem__"] = [func_to_add] else: # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the # protocol defined in collections module) whereas the typing module consider it should not - #  We do not want __class_getitem__ to be found in the classdef + # We do not want __class_getitem__ to be found in the classdef _forbid_class_getitem_access(res) else: - #  Within python3.9 discrepencies exist between some collections.abc containers that are subscriptable whereas - #  corresponding containers in the typing module are not! This is the case at least for ByteString. - #  It is far more to complex and dangerous to try to remove __class_getitem__ method from all the ancestors of the - #  current class. Instead we raise an AttributeInferenceError if we try to access it. + # Within python3.9 discrepencies exist between some collections.abc containers that are subscriptable whereas + # corresponding containers in the typing module are not! This is the case at least for ByteString. + # It is far more to complex and dangerous to try to remove __class_getitem__ method from all the ancestors of the + # current class. Instead we raise an AttributeInferenceError if we try to access it. maybe_type_var = node.args[1] if isinstance(maybe_type_var, nodes.Const) and maybe_type_var.value == 0: - #  Starting with Python39 the _alias function is in fact instantiation of _SpecialGenericAlias class. - #  Thus the type is not Generic if the second argument of the call is equal to zero + # Starting with Python39 the _alias function is in fact instantiation of _SpecialGenericAlias class. + # Thus the type is not Generic if the second argument of the call is equal to zero _forbid_class_getitem_access(res) return iter([res]) return iter([astroid.Uninferable]) diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index 3c2b73370b..dd5aa1257a 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -2621,13 +2621,13 @@ def getitem(self, index, context=None): except exceptions.AttributeInferenceError as exc: if isinstance(self, ClassDef): # subscripting a class definition may be - #  achieved thanks to __class_getitem__ method - #  which is a classmethod defined in the class - #  that supports subscript and not in the metaclass + # achieved thanks to __class_getitem__ method + # which is a classmethod defined in the class + # that supports subscript and not in the metaclass try: methods = self.getattr("__class_getitem__") # Here it is assumed that the __class_getitem__ node is - #  a FunctionDef. One possible improvement would be to deal + # a FunctionDef. One possible improvement would be to deal # with more generic inference. except exceptions.AttributeInferenceError: raise exceptions.AstroidTypeError( @@ -2647,9 +2647,9 @@ def getitem(self, index, context=None): except AttributeError: # Starting with python3.9, builtin types list, dict etc... # are subscriptable thanks to __class_getitem___ classmethod. - #  However in such case the method is bound to an EmptyNode and - #  EmptyNode doesn't have infer_call_result method yielding to - #  AttributeError + # However in such case the method is bound to an EmptyNode and + # EmptyNode doesn't have infer_call_result method yielding to + # AttributeError if ( isinstance(method, node_classes.EmptyNode) and self.name in ("list", "dict", "set", "tuple", "frozenset") diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index d80dee6699..57b9dc0910 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1053,13 +1053,11 @@ class CollectionsBrain(unittest.TestCase): def test_collections_object_not_subscriptable(self): """ Test that unsubscriptable types are detected - Hashable is not subscriptable even with python39 """ wrong_node = builder.extract_node( """ import collections.abc - collections.abc.Hashable[int] """ ) @@ -1068,7 +1066,6 @@ def test_collections_object_not_subscriptable(self): right_node = builder.extract_node( """ import collections.abc - collections.abc.Hashable """ ) @@ -1090,7 +1087,6 @@ def test_collections_object_subscriptable(self): right_node = builder.extract_node( """ import collections.abc - collections.abc.MutableSet[int] """ ) @@ -1115,13 +1111,12 @@ def test_collections_object_subscriptable(self): @test_utils.require_version(maxver="3.9") def test_collections_object_not_yet_subscriptable(self): """ - Test that unsubscriptable types are detected as so. + Test that unsubscriptable types are detected as such. Until python39 MutableSet of the collections module is not subscriptable. """ wrong_node = builder.extract_node( """ import collections.abc - collections.abc.MutableSet[int] """ ) @@ -1130,7 +1125,6 @@ def test_collections_object_not_yet_subscriptable(self): right_node = builder.extract_node( """ import collections.abc - collections.abc.MutableSet """ ) @@ -1157,7 +1151,6 @@ def test_collections_object_subscriptable_2(self): node = builder.extract_node( """ import collections.abc - class Derived(collections.abc.Iterator[int]): pass """ @@ -1180,7 +1173,6 @@ def test_collections_object_not_yet_subscriptable_2(self): node = builder.extract_node( """ import collections.abc - collections.abc.Iterator[int] """ ) @@ -1193,7 +1185,6 @@ def test_collections_object_subscriptable_3(self): right_node = builder.extract_node( """ import collections.abc - collections.abc.ByteString[int] """ ) @@ -1438,8 +1429,8 @@ class Derived2(typing.OrderedDict[int, str]): """ ) inferred = next(node.infer()) - #  OrderedDict has no metaclass because it - #  inherits from dict which is C coded + # OrderedDict has no metaclass because it + # inherits from dict which is C coded self.assertIsNone(inferred.metaclass()) assertEqualMro( inferred, @@ -1456,7 +1447,6 @@ def test_typing_object_not_subscriptable(self): wrong_node = builder.extract_node( """ import typing - typing.Hashable[int] """ ) @@ -1465,7 +1455,6 @@ def test_typing_object_not_subscriptable(self): right_node = builder.extract_node( """ import typing - typing.Hashable """ ) @@ -1487,7 +1476,6 @@ def test_typing_object_subscriptable(self): right_node = builder.extract_node( """ import typing - typing.MutableSet[int] """ ) @@ -1515,7 +1503,6 @@ def test_typing_object_notsubscriptable_3(self): right_node = builder.extract_node( """ import typing - typing.ByteString """ ) @@ -1531,7 +1518,7 @@ def test_typing_object_builtin_subscriptable(self): """ Test that builtins alias, such as typing.List, are subscriptable """ - #  Do not test Tuple as it is inferred as _TupleType class (needs a brain?) + # Do not test Tuple as it is inferred as _TupleType class (needs a brain?) for typename in ("List", "Dict", "Set", "FrozenSet"): src = """ import typing