Fix crash in daemon mode on new import cycle (#14508)

Fixes #14329

This fixes the second crash reported in the issue (other one is already
fixed). This one is tricky, it looks like it happens only when we bring
in a new import cycle in an incremental update with
`--follow-import=normal`. To explain the reason, a little reminder of
how semantic analyzer passes work:

* Originally, we recorded progress automatically when some new symbol
was resolved and added to symbol tables.
* With implementation of recursive types, this mechanism was
insufficient, as recursive types require modifying some symbols _in
place_, this is why `force_progress` flag was added to `defer()`.
* I was only careful with this flag for recursive type aliases (that
were implemented first), for other things (like recursive TypedDicts,
etc) I just always passed `force_progress=True` (without checking if we
actually resolved some placeholder types).
* The reasoning for that is if we ever add `becomes_typeinfo=True`,
there is no way this symbol will later be unresolved (otherwise how
would we know this is something that is a type).
* It turns out this reasoning doesn't work in some edge cases in daemon
mode, we do put some placeholders with `becomes_typeinfo=True` for
symbols imported from modules that were not yet processed, thus causing
a crash (see test cases).
* There were two options to fix this: one is to stop creating
placeholders with `becomes_typeinfo=True` for unimported symbols in
daemon mode, other one is to always carefully check if in-place update
of a symbol actually resulted in progress.
* Ultimately I decided that the first way is too fragile (and I don't
understand how import following works for daemon anyway), and the second
way is something that is technically correct anyway, so here is this PR

I didn't add test cases for each of the crash scenarios, since they are
all very similar. I only added two that I encountered "in the wild",
upper bound and tuple base caused actual crash in `trio` stubs, plus
also randomly a test for a TypedDict crash.

_EDIT:_ and one more thing, the "cannot resolve name" error should never
appear in normal mode, only in daemon update (see reasoning above), so I
don't make those error messages detailed, just add some minimal info if
we will need to debug them.

Author: ilevkivskyi

mypy/semanal.py

@@ -1049,7 +1049,12 @@ def setup_self_type(self) -> None:
         if info.self_type is not None:
             if has_placeholder(info.self_type.upper_bound):
                 # Similar to regular (user defined) type variables.
-                self.defer(force_progress=True)
+                self.process_placeholder(
+                    None,
+                    "Self upper bound",
+                    info,
+                    force_progress=info.self_type.upper_bound != fill_typevars(info),
+                )
             else:
                 return
         info.self_type = TypeVarType("Self", f"{info.fullname}.Self", 0, [], fill_typevars(info))
@@ -2132,7 +2137,9 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc
             self.fail("Class has two incompatible bases derived from tuple", defn)
             defn.has_incompatible_baseclass = True
         if info.special_alias and has_placeholder(info.special_alias.target):
-            self.defer(force_progress=True)
+            self.process_placeholder(
+                None, "tuple base", defn, force_progress=base != info.tuple_type
+            )
         info.update_tuple_type(base)
         self.setup_alias_type_vars(defn)
 
@@ -3913,12 +3920,16 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> bool:
             type_var = TypeVarExpr(name, self.qualified_name(name), values, upper_bound, variance)
             type_var.line = call.line
             call.analyzed = type_var
+            updated = True
         else:
             assert isinstance(call.analyzed, TypeVarExpr)
+            updated = values != call.analyzed.values or upper_bound != call.analyzed.upper_bound
             call.analyzed.upper_bound = upper_bound
             call.analyzed.values = values
         if any(has_placeholder(v) for v in values) or has_placeholder(upper_bound):
-            self.defer(force_progress=True)
+            self.process_placeholder(
+                None, f"TypeVar {'values' if values else 'upper bound'}", s, force_progress=updated
+            )
 
         self.add_symbol(name, call.analyzed, s)
         return True
@@ -5931,7 +5942,9 @@ def is_incomplete_namespace(self, fullname: str) -> bool:
         """
         return fullname in self.incomplete_namespaces
 
-    def process_placeholder(self, name: str, kind: str, ctx: Context) -> None:
+    def process_placeholder(
+        self, name: str | None, kind: str, ctx: Context, force_progress: bool = False
+    ) -> None:
         """Process a reference targeting placeholder node.
 
         If this is not a final iteration, defer current node,
@@ -5943,10 +5956,11 @@ def process_placeholder(self, name: str, kind: str, ctx: Context) -> None:
         if self.final_iteration:
             self.cannot_resolve_name(name, kind, ctx)
         else:
-            self.defer(ctx)
+            self.defer(ctx, force_progress=force_progress)
 
-    def cannot_resolve_name(self, name: str, kind: str, ctx: Context) -> None:
-        self.fail(f'Cannot resolve {kind} "{name}" (possible cyclic definition)', ctx)
+    def cannot_resolve_name(self, name: str | None, kind: str, ctx: Context) -> None:
+        name_format = f' "{name}"' if name else ""
+        self.fail(f"Cannot resolve {kind}{name_format} (possible cyclic definition)", ctx)
         if not self.options.disable_recursive_aliases and self.is_func_scope():
             self.note("Recursive types are not allowed at function scope", ctx)
 

mypy/semanal_namedtuple.py

@@ -501,7 +501,9 @@ def build_namedtuple_typeinfo(
         info.is_named_tuple = True
         tuple_base = TupleType(types, fallback)
         if info.special_alias and has_placeholder(info.special_alias.target):
-            self.api.defer(force_progress=True)
+            self.api.process_placeholder(
+                None, "NamedTuple item", info, force_progress=tuple_base != info.tuple_type
+            )
         info.update_tuple_type(tuple_base)
         info.line = line
         # For use by mypyc.

mypy/semanal_newtype.py

@@ -249,10 +249,16 @@ def build_newtype_typeinfo(
         init_func = FuncDef("__init__", args, Block([]), typ=signature)
         init_func.info = info
         init_func._fullname = info.fullname + ".__init__"
+        if not existing_info:
+            updated = True
+        else:
+            previous_sym = info.names["__init__"].node
+            assert isinstance(previous_sym, FuncDef)
+            updated = old_type != previous_sym.arguments[1].variable.type
         info.names["__init__"] = SymbolTableNode(MDEF, init_func)
 
-        if has_placeholder(old_type) or info.tuple_type and has_placeholder(info.tuple_type):
-            self.api.defer(force_progress=True)
+        if has_placeholder(old_type):
+            self.api.process_placeholder(None, "NewType base", info, force_progress=updated)
         return info
 
     # Helpers

mypy/semanal_shared.py

@@ -232,6 +232,12 @@ def qualified_name(self, n: str) -> str:
     def is_typeshed_stub_file(self) -> bool:
         raise NotImplementedError
 
+    @abstractmethod
+    def process_placeholder(
+        self, name: str | None, kind: str, ctx: Context, force_progress: bool = False
+    ) -> None:
+        raise NotImplementedError
+
 
 def set_callable_name(sig: Type, fdef: FuncDef) -> ProperType:
     sig = get_proper_type(sig)

mypy/semanal_typeddict.py

@@ -535,7 +535,9 @@ def build_typeddict_typeinfo(
         info = existing_info or self.api.basic_new_typeinfo(name, fallback, line)
         typeddict_type = TypedDictType(dict(zip(items, types)), required_keys, fallback)
         if info.special_alias and has_placeholder(info.special_alias.target):
-            self.api.defer(force_progress=True)
+            self.api.process_placeholder(
+                None, "TypedDict item", info, force_progress=typeddict_type != info.typeddict_type
+            )
         info.update_typeddict_type(typeddict_type)
         return info
 

mypy/types.py

@@ -2857,6 +2857,14 @@ def accept(self, visitor: TypeVisitor[T]) -> T:
         assert isinstance(visitor, SyntheticTypeVisitor)
         return cast(T, visitor.visit_placeholder_type(self))
 
+    def __hash__(self) -> int:
+        return hash((self.fullname, tuple(self.args)))
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, PlaceholderType):
+            return NotImplemented
+        return self.fullname == other.fullname and self.args == other.args
+
     def serialize(self) -> str:
         # We should never get here since all placeholders should be replaced
         # during semantic analysis.

test-data/unit/fine-grained-follow-imports.test

@@ -769,3 +769,80 @@ from . import mod3
 ==
 main.py:1: error: Cannot find implementation or library stub for module named "pkg"
 main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+
+[case testNewImportCycleTypeVarBound]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+# cmd2: mypy other.py
+
+[file main.py]
+# empty
+
+[file other.py.2]
+import trio
+
+[file trio/__init__.py.2]
+from typing import TypeVar
+import trio
+from . import abc as abc
+
+T = TypeVar("T", bound=trio.abc.A)
+
+[file trio/abc.py.2]
+import trio
+class A: ...
+[out]
+==
+
+[case testNewImportCycleTupleBase]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+# cmd2: mypy other.py
+
+[file main.py]
+# empty
+
+[file other.py.2]
+import trio
+
+[file trio/__init__.py.2]
+from typing import TypeVar, Tuple
+import trio
+from . import abc as abc
+
+class C(Tuple[trio.abc.A, trio.abc.A]): ...
+
+[file trio/abc.py.2]
+import trio
+class A: ...
+[builtins fixtures/tuple.pyi]
+[out]
+==
+
+[case testNewImportCycleTypedDict]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+# cmd2: mypy other.py
+
+[file main.py]
+# empty
+
+[file other.py.2]
+import trio
+
+[file trio/__init__.py.2]
+from typing import TypeVar
+from typing_extensions import TypedDict
+import trio
+from . import abc as abc
+
+class C(TypedDict):
+    x: trio.abc.A
+    y: trio.abc.A
+
+[file trio/abc.py.2]
+import trio
+class A: ...
+[builtins fixtures/dict.pyi]
+[out]
+==