From 40868483c9ebca6b1003ca929c4766d6fa9f4606 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 12 Jul 2017 16:49:46 +0200 Subject: [PATCH 01/11] Hide imported names in stubs unless 'as id' is used (PEP 484) --- mypy/nodes.py | 10 +++++- mypy/semanal.py | 37 ++++++++++--------- test-data/unit/check-modules.test | 60 ++++++++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 19 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 6cc75f5b8f22..ca1bb96b54df 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2274,6 +2274,8 @@ class SymbolTableNode: # If False, this name won't be imported via 'from import *'. # This has no effect on names within classes. module_public = True + # If true, the name will be never exported (neede for stub files) + module_hidden = False # For deserialized MODULE_REF nodes, the referenced module name; # for other nodes, optionally the name of the referenced object. cross_ref = None # type: Optional[str] @@ -2283,11 +2285,13 @@ class SymbolTableNode: def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None, typ: 'mypy.types.Type' = None, module_public: bool = True, normalized: bool = False, - alias_tvars: Optional[List[str]] = None) -> None: + alias_tvars: Optional[List[str]] = None, + module_hidden: bool = False) -> None: self.kind = kind self.node = node self.type_override = typ self.mod_id = mod_id + self.module_hidden = module_hidden self.module_public = module_public self.normalized = normalized self.alias_tvars = alias_tvars @@ -2332,6 +2336,8 @@ def serialize(self, prefix: str, name: str) -> JsonDict: data = {'.class': 'SymbolTableNode', 'kind': node_kinds[self.kind], } # type: JsonDict + if self.module_hidden: + data['module_hidden'] = True if not self.module_public: data['module_public'] = False if self.kind == MODULE_REF: @@ -2369,6 +2375,8 @@ def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': stnode = SymbolTableNode(kind, node, typ=typ) if 'alias_tvars' in data: stnode.alias_tvars = data['alias_tvars'] + if 'module_hidden' in data: + stnode.module_hidden = data['module_hidden'] if 'module_public' in data: stnode.module_public = data['module_public'] return stnode diff --git a/mypy/semanal.py b/mypy/semanal.py index bcbe495e1c93..4c8421366796 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1317,15 +1317,15 @@ def visit_import(self, i: Import) -> None: if as_id is not None: self.add_module_symbol(id, as_id, module_public=True, context=i) else: - # Modules imported in a stub file without using 'as x' won't get exported when - # doing 'from m import *'. - module_public = not self.is_stub_file + # Modules imported in a stub file without using 'as x' won't get exported + module_hidden = self.is_stub_file base = id.split('.')[0] - self.add_module_symbol(base, base, module_public=module_public, - context=i) - self.add_submodules_to_parent_modules(id, module_public) + self.add_module_symbol(base, base, module_public=not module_hidden, + context=i, module_hidden=module_hidden) + self.add_submodules_to_parent_modules(id, not module_hidden, module_hidden=module_hidden) - def add_submodules_to_parent_modules(self, id: str, module_public: bool) -> None: + def add_submodules_to_parent_modules(self, id: str, module_public: bool, + module_hidden: bool = False) -> None: """Recursively adds a reference to a newly loaded submodule to its parent. When you import a submodule in any way, Python will add a reference to that @@ -1346,16 +1346,18 @@ def add_submodules_to_parent_modules(self, id: str, module_public: bool) -> None child_mod = self.modules.get(id) if child_mod: sym = SymbolTableNode(MODULE_REF, child_mod, parent, - module_public=module_public) + module_public=module_public, + module_hidden=module_hidden) parent_mod.names[child] = sym id = parent def add_module_symbol(self, id: str, as_id: str, module_public: bool, - context: Context) -> None: + context: Context, module_hidden: bool = False) -> None: if id in self.modules: m = self.modules[id] self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id, - module_public=module_public), context) + module_public=module_public, + module_hidden=module_hidden), context) else: self.add_unknown_symbol(as_id, context, is_import=True) @@ -1378,7 +1380,7 @@ def visit_import_from(self, imp: ImportFrom) -> None: elif possible_module_id in self.missing_modules: missing = True - if node and node.kind != UNBOUND_IMPORTED: + if node and node.kind != UNBOUND_IMPORTED and not node.module_hidden: node = self.normalize_type_alias(node, imp) if not node: return @@ -1390,13 +1392,14 @@ def visit_import_from(self, imp: ImportFrom) -> None: imported_id, existing_symbol, node, imp): continue # 'from m import x as x' exports x in a stub file. - module_public = not self.is_stub_file or as_id is not None + module_hidden = self.is_stub_file and as_id is None symbol = SymbolTableNode(node.kind, node.node, self.cur_mod_id, node.type_override, - module_public=module_public, + module_public=not module_hidden, normalized=node.normalized, - alias_tvars=node.alias_tvars) + alias_tvars=node.alias_tvars, + module_hidden=module_hidden) self.add_symbol(imported_id, symbol, imp) elif module and not missing: # Missing attribute. @@ -3145,7 +3148,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None: # bar in its namespace. This must be done for all types of bar. file = cast(Optional[MypyFile], base.node) # can't use isinstance due to issue #2999 n = file.names.get(expr.name, None) if file is not None else None - if n: + if n and not n.module_hidden: n = self.normalize_type_alias(n, expr) if not n: return @@ -3458,12 +3461,12 @@ def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: elif isinstance(n.node, MypyFile): n = n.node.names.get(parts[i], None) # TODO: What if node is Var or FuncDef? - if not n: + if not n or n.module_hidden: self.name_not_defined(name, ctx) break if n: n = self.normalize_type_alias(n, ctx) - return n + return n if n and not n.module_hidden else None def builtin_type(self, fully_qualified_name: str) -> Instance: sym = self.lookup_fully_qualified(fully_qualified_name) diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index 4a76ad3704e9..fc24efbe9fa0 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -960,7 +960,7 @@ from other import y # Disallowed x + '' # Error here y + '' # But not here [file stub.pyi] -from non_stub import x +from non_stub import x as x [file non_stub.py] x = 42 [file other.py] @@ -1640,3 +1640,61 @@ m = n # E: Cannot assign multiple modules to name 'm' without explicit 'types.M [file n.py] [builtins fixtures/module.pyi] + +[case testNoReExportFromStubs] +from stub import Iterable # E: Module 'stub' has no attribute 'Iterable' +from stub import C + +c = C() +reveal_type(c.x) # E: Revealed type is 'builtins.int' +it: Iterable[int] +reveal_type(it) # E: Revealed type is 'Any' + +[file stub.pyi] +from typing import Iterable +from substub import C as C + +def fun(x: Iterable[str]) -> Iterable[int]: pass + +[file substub.pyi] +class C: + x: int + +[builtins fixtures/module.pyi] + +[case testNoReExportFromStubsMemberType] +import stub + +c = stub.C() +reveal_type(c.x) # E: Revealed type is 'builtins.int' +it: stub.Iterable[int] # E: Name 'stub.Iterable' is not defined +reveal_type(it) # E: Revealed type is 'Any' + +[file stub.pyi] +from typing import Iterable +from substub import C as C + +def fun(x: Iterable[str]) -> Iterable[int]: pass + +[file substub.pyi] +class C: + x: int + +[builtins fixtures/module.pyi] + +[case testNoReExportFromStubsMemberVar] +import stub + +reveal_type(stub.y) # E: Revealed type is 'builtins.int' +reveal_type(stub.z) # E: Revealed type is 'Any' \ + # E: Module has no attribute "z" + +[file stub.pyi] +from substub import y as y +from substub import z + +[file substub.pyi] +y = 42 +z: int + +[builtins fixtures/module.pyi] From bd2536b8f2918e75251c56a08c3fc881b24e2d3c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 12 Jul 2017 17:38:39 +0200 Subject: [PATCH 02/11] Fix spelling in comment --- mypy/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index ca1bb96b54df..db3f472b4dbc 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2274,7 +2274,7 @@ class SymbolTableNode: # If False, this name won't be imported via 'from import *'. # This has no effect on names within classes. module_public = True - # If true, the name will be never exported (neede for stub files) + # If True, the name will be never exported (needed for stub files) module_hidden = False # For deserialized MODULE_REF nodes, the referenced module name; # for other nodes, optionally the name of the referenced object. From c667d7991ab4bfe6a07d564902ad1b0df928201e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 12 Jul 2017 18:51:33 +0200 Subject: [PATCH 03/11] Fix logic with hidden nodes --- mypy/semanal.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 4c8421366796..def01f25db8b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3148,7 +3148,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None: # bar in its namespace. This must be done for all types of bar. file = cast(Optional[MypyFile], base.node) # can't use isinstance due to issue #2999 n = file.names.get(expr.name, None) if file is not None else None - if n and not n.module_hidden: + if n and (not n.module_hidden or isinstance(n.node, MypyFile)): n = self.normalize_type_alias(n, expr) if not n: return @@ -3461,11 +3461,13 @@ def lookup_qualified(self, name: str, ctx: Context) -> SymbolTableNode: elif isinstance(n.node, MypyFile): n = n.node.names.get(parts[i], None) # TODO: What if node is Var or FuncDef? - if not n or n.module_hidden: + if not n: self.name_not_defined(name, ctx) break if n: n = self.normalize_type_alias(n, ctx) + if n and n.module_hidden: + self.name_not_defined(name, ctx) return n if n and not n.module_hidden else None def builtin_type(self, fully_qualified_name: str) -> Instance: From 8e51825a12a616790cfea9d44316d4c9e8c4e2b7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 12 Jul 2017 19:07:38 -0700 Subject: [PATCH 04/11] Fix lint in meantime --- mypy/semanal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index def01f25db8b..17c35b3ee70a 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1322,7 +1322,8 @@ def visit_import(self, i: Import) -> None: base = id.split('.')[0] self.add_module_symbol(base, base, module_public=not module_hidden, context=i, module_hidden=module_hidden) - self.add_submodules_to_parent_modules(id, not module_hidden, module_hidden=module_hidden) + self.add_submodules_to_parent_modules(id, not module_hidden, + module_hidden=module_hidden) def add_submodules_to_parent_modules(self, id: str, module_public: bool, module_hidden: bool = False) -> None: From 8d9d5e26f7997579c1cf7bdcdc915c6ffe4a6bd5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 13 Jul 2017 05:21:25 -0700 Subject: [PATCH 05/11] Always re-export child modules --- mypy/semanal.py | 10 +++---- test-data/unit/check-modules.test | 45 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index ca05a3afbe78..b46ab0006c2c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1322,8 +1322,7 @@ def visit_import(self, i: Import) -> None: base = id.split('.')[0] self.add_module_symbol(base, base, module_public=not module_hidden, context=i, module_hidden=module_hidden) - self.add_submodules_to_parent_modules(id, not module_hidden, - module_hidden=module_hidden) + self.add_submodules_to_parent_modules(id, True) def add_submodules_to_parent_modules(self, id: str, module_public: bool, module_hidden: bool = False) -> None: @@ -1347,8 +1346,7 @@ def add_submodules_to_parent_modules(self, id: str, module_public: bool, child_mod = self.modules.get(id) if child_mod: sym = SymbolTableNode(MODULE_REF, child_mod, parent, - module_public=module_public, - module_hidden=module_hidden) + module_public=module_public) parent_mod.names[child] = sym id = parent @@ -1369,11 +1367,11 @@ def visit_import_from(self, imp: ImportFrom) -> None: for id, as_id in imp.names: node = module.names.get(id) if module else None missing = False + possible_module_id = import_id + '.' + id # If the module does not contain a symbol with the name 'id', # try checking if it's a module instead. if not node or node.kind == UNBOUND_IMPORTED: - possible_module_id = import_id + '.' + id mod = self.modules.get(possible_module_id) if mod is not None: node = SymbolTableNode(MODULE_REF, mod, import_id) @@ -1393,7 +1391,7 @@ def visit_import_from(self, imp: ImportFrom) -> None: imported_id, existing_symbol, node, imp): continue # 'from m import x as x' exports x in a stub file. - module_hidden = self.is_stub_file and as_id is None + module_hidden = self.is_stub_file and as_id is None and possible_module_id not in self.modules symbol = SymbolTableNode(node.kind, node.node, self.cur_mod_id, node.type_override, diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index fc24efbe9fa0..4d6a8915b3e5 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -1698,3 +1698,48 @@ y = 42 z: int [builtins fixtures/module.pyi] + +[case testReExportChildStubs] +import mod +from mod import submod + +reveal_type(mod.x) # E: Revealed type is 'mod.submod.C' +y = submod.C() +reveal_type(y.a) # E: Revealed type is 'builtins.str' + +[file mod/__init__.pyi] +from . import submod +x: submod.C + +[file mod/submod.pyi] +class C: + a: str + +[builtins fixtures/module.pyi] + +[case testReExportChildStubs2] +import mod.submod + +y = mod.submod.C() +reveal_type(y.a) # E: Revealed type is 'builtins.str' + +[file mod/__init__.pyi] +from . import submod +x: submod.C + +[file mod/submod.pyi] +class C: + a: str + +[builtins fixtures/module.pyi] + +[case testNoReExportNestedStub] +from stub import substub # E: Module 'stub' has no attribute 'substub' + +[file stub.pyi] +import substub + +[file substub.pyi] +x = 42 + +[file mod/submod.pyi] From f9f4ad727cc4ffe6a8bf50619f2d3316e7d3f9a0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 13 Jul 2017 05:22:37 -0700 Subject: [PATCH 06/11] Remove unnecessary parameter --- mypy/semanal.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index b46ab0006c2c..cf7165de73f1 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1324,8 +1324,7 @@ def visit_import(self, i: Import) -> None: context=i, module_hidden=module_hidden) self.add_submodules_to_parent_modules(id, True) - def add_submodules_to_parent_modules(self, id: str, module_public: bool, - module_hidden: bool = False) -> None: + def add_submodules_to_parent_modules(self, id: str, module_public: bool) -> None: """Recursively adds a reference to a newly loaded submodule to its parent. When you import a submodule in any way, Python will add a reference to that From eb23885a8d78a914edb96a9c074b6b68909ef8d3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 13 Jul 2017 05:38:44 -0700 Subject: [PATCH 07/11] Fix lint --- mypy/semanal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index cf7165de73f1..89eb4846c538 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1390,7 +1390,8 @@ def visit_import_from(self, imp: ImportFrom) -> None: imported_id, existing_symbol, node, imp): continue # 'from m import x as x' exports x in a stub file. - module_hidden = self.is_stub_file and as_id is None and possible_module_id not in self.modules + module_hidden = (self.is_stub_file and as_id is None and + possible_module_id not in self.modules) symbol = SymbolTableNode(node.kind, node.node, self.cur_mod_id, node.type_override, From 191a1c0165d125aeec726615c584d515e3c04aec Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 15 Jul 2017 07:08:27 -0700 Subject: [PATCH 08/11] Add one more test (+minor simplification) --- mypy/semanal.py | 2 +- test-data/unit/check-modules.test | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 89eb4846c538..d34b6d7cfeb3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3148,7 +3148,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None: # bar in its namespace. This must be done for all types of bar. file = cast(Optional[MypyFile], base.node) # can't use isinstance due to issue #2999 n = file.names.get(expr.name, None) if file is not None else None - if n and (not n.module_hidden or isinstance(n.node, MypyFile)): + if n and not n.module_hidden: n = self.normalize_type_alias(n, expr) if not n: return diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index 4d6a8915b3e5..3097a3685c03 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -1733,6 +1733,25 @@ class C: [builtins fixtures/module.pyi] +[case testNoReExportChildStubs] +import mod +from mod import C, D # E: Module 'mod' has no attribute 'C' + +reveal_type(mod.x) # E: Revealed type is 'mod.submod.C' +mod.C # E: Module has no attribute "C" +y = mod.D() +reveal_type(y.a) # E: Revealed type is 'builtins.str' + +[file mod/__init__.pyi] +from .submod import C, D as D +x: C + +[file mod/submod.pyi] +class C: pass +class D: + a: str +[builtins fixtures/module.pyi] + [case testNoReExportNestedStub] from stub import substub # E: Module 'stub' has no attribute 'substub' From 1e1001e1ec713506f9dfc0662d4653d03cf0e3e7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 15 Jul 2017 07:21:32 -0700 Subject: [PATCH 09/11] More minor simplifications --- mypy/semanal.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d34b6d7cfeb3..62edc328caa7 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1318,11 +1318,11 @@ def visit_import(self, i: Import) -> None: self.add_module_symbol(id, as_id, module_public=True, context=i) else: # Modules imported in a stub file without using 'as x' won't get exported - module_hidden = self.is_stub_file + module_public = not self.is_stub_file base = id.split('.')[0] - self.add_module_symbol(base, base, module_public=not module_hidden, - context=i, module_hidden=module_hidden) - self.add_submodules_to_parent_modules(id, True) + self.add_module_symbol(base, base, module_public=module_public, + context=i, module_hidden=not module_public) + self.add_submodules_to_parent_modules(id, module_public) def add_submodules_to_parent_modules(self, id: str, module_public: bool) -> None: """Recursively adds a reference to a newly loaded submodule to its parent. @@ -1390,8 +1390,8 @@ def visit_import_from(self, imp: ImportFrom) -> None: imported_id, existing_symbol, node, imp): continue # 'from m import x as x' exports x in a stub file. - module_hidden = (self.is_stub_file and as_id is None and - possible_module_id not in self.modules) + module_public = not self.is_stub_file or as_id is not None + module_hidden = not module_public and possible_module_id not in self.modules symbol = SymbolTableNode(node.kind, node.node, self.cur_mod_id, node.type_override, From 92684bdd4cd0089ce32074323ac2d16a6f0d4e81 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 15 Jul 2017 07:24:50 -0700 Subject: [PATCH 10/11] Don't mix the flags --- mypy/semanal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 62edc328caa7..4ed563c427c8 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1395,7 +1395,7 @@ def visit_import_from(self, imp: ImportFrom) -> None: symbol = SymbolTableNode(node.kind, node.node, self.cur_mod_id, node.type_override, - module_public=not module_hidden, + module_public=module_public, normalized=node.normalized, alias_tvars=node.alias_tvars, module_hidden=module_hidden) From 9d7d0c768ad8cd5518d18d6f6d9e809b1a21f950 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 31 Jul 2017 00:35:20 +0200 Subject: [PATCH 11/11] Address CR --- mypy/semanal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 5927d006be67..2969b8a8cfc3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3512,7 +3512,9 @@ def lookup_qualified(self, name: str, ctx: Context, n = self.normalize_type_alias(n, ctx) if n and n.module_hidden: self.name_not_defined(name, ctx) - return n if n and not n.module_hidden else None + if n and not n.module_hidden: + return n + return None def builtin_type(self, fully_qualified_name: str) -> Instance: sym = self.lookup_fully_qualified(fully_qualified_name)