Fix crash in daemon mode on new import cycle (#14508)

Fixes #14329

This fixes the second crash reported in the issue (other one is already
fixed). This one is tricky, it looks like it happens only when we bring
in a new import cycle in an incremental update with
`--follow-import=normal`. To explain the reason, a little reminder of
how semantic analyzer passes work:

* Originally, we recorded progress automatically when some new symbol
was resolved and added to symbol tables.
* With implementation of recursive types, this mechanism was
insufficient, as recursive types require modifying some symbols _in
place_, this is why `force_progress` flag was added to `defer()`.
* I was only careful with this flag for recursive type aliases (that
were implemented first), for other things (like recursive TypedDicts,
etc) I just always passed `force_progress=True` (without checking if we
actually resolved some placeholder types).
* The reasoning for that is if we ever add `becomes_typeinfo=True`,
there is no way this symbol will later be unresolved (otherwise how
would we know this is something that is a type).
* It turns out this reasoning doesn't work in some edge cases in daemon
mode, we do put some placeholders with `becomes_typeinfo=True` for
symbols imported from modules that were not yet processed, thus causing
a crash (see test cases).
* There were two options to fix this: one is to stop creating
placeholders with `becomes_typeinfo=True` for unimported symbols in
daemon mode, other one is to always carefully check if in-place update
of a symbol actually resulted in progress.
* Ultimately I decided that the first way is too fragile (and I don't
understand how import following works for daemon anyway), and the second
way is something that is technically correct anyway, so here is this PR

I didn't add test cases for each of the crash scenarios, since they are
all very similar. I only added two that I encountered "in the wild",
upper bound and tuple base caused actual crash in `trio` stubs, plus
also randomly a test for a TypedDict crash.

_EDIT:_ and one more thing, the "cannot resolve name" error should never
appear in normal mode, only in daemon update (see reasoning above), so I
don't make those error messages detailed, just add some minimal info if
we will need to debug them.

mypy/semanal.py

@@ -1049,7 +1049,12 @@ def setup_self_type(self) -> None:
  if info.self_type is not None:
  if has_placeholder(info.self_type.upper_bound):
  # Similar to regular (user defined) type variables.
- self.defer(force_progress=True)
+ self.process_placeholder(
+ None,
+ "Self upper bound",
+ info,
+ force_progress=info.self_type.upper_bound != fill_typevars(info),
+ )
  else:
  return
  info.self_type = TypeVarType("Self", f"{info.fullname}.Self", 0, [], fill_typevars(info))
@@ -2132,7 +2137,9 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc
  self.fail("Class has two incompatible bases derived from tuple", defn)
  defn.has_incompatible_baseclass = True
  if info.special_alias and has_placeholder(info.special_alias.target):
- self.defer(force_progress=True)
+ self.process_placeholder(
+ None, "tuple base", defn, force_progress=base != info.tuple_type
+ )
  info.update_tuple_type(base)
  self.setup_alias_type_vars(defn)
 
@@ -3913,12 +3920,16 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> bool:
  type_var = TypeVarExpr(name, self.qualified_name(name), values, upper_bound, variance)
  type_var.line = call.line
  call.analyzed = type_var
+ updated = True
  else:
  assert isinstance(call.analyzed, TypeVarExpr)
+ updated = values != call.analyzed.values or upper_bound != call.analyzed.upper_bound
  call.analyzed.upper_bound = upper_bound
  call.analyzed.values = values
  if any(has_placeholder(v) for v in values) or has_placeholder(upper_bound):
- self.defer(force_progress=True)
+ self.process_placeholder(
+ None, f"TypeVar {'values' if values else 'upper bound'}", s, force_progress=updated
+ )
 
  self.add_symbol(name, call.analyzed, s)
  return True
@@ -5931,7 +5942,9 @@ def is_incomplete_namespace(self, fullname: str) -> bool:
  """
  return fullname in self.incomplete_namespaces
 
- def process_placeholder(self, name: str, kind: str, ctx: Context) -> None:
+ def process_placeholder(
+ self, name: str | None, kind: str, ctx: Context, force_progress: bool = False
+ ) -> None:
  """Process a reference targeting placeholder node.
 
  If this is not a final iteration, defer current node,
@@ -5943,10 +5956,11 @@ def process_placeholder(self, name: str, kind: str, ctx: Context) -> None:
  if self.final_iteration:
  self.cannot_resolve_name(name, kind, ctx)
  else:
- self.defer(ctx)
+ self.defer(ctx, force_progress=force_progress)
 
- def cannot_resolve_name(self, name: str, kind: str, ctx: Context) -> None:
- self.fail(f'Cannot resolve {kind} "{name}" (possible cyclic definition)', ctx)
+ def cannot_resolve_name(self, name: str | None, kind: str, ctx: Context) -> None:
+ name_format = f' "{name}"' if name else ""
+ self.fail(f"Cannot resolve {kind}{name_format} (possible cyclic definition)", ctx)
  if not self.options.disable_recursive_aliases and self.is_func_scope():
  self.note("Recursive types are not allowed at function scope", ctx)
 

mypy/semanal_namedtuple.py

@@ -501,7 +501,9 @@ def build_namedtuple_typeinfo(
  info.is_named_tuple = True
  tuple_base = TupleType(types, fallback)
  if info.special_alias and has_placeholder(info.special_alias.target):
- self.api.defer(force_progress=True)
+ self.api.process_placeholder(
+ None, "NamedTuple item", info, force_progress=tuple_base != info.tuple_type
+ )
  info.update_tuple_type(tuple_base)
  info.line = line
  # For use by mypyc.

mypy/semanal_newtype.py

@@ -249,10 +249,16 @@ def build_newtype_typeinfo(
  init_func = FuncDef("__init__", args, Block([]), typ=signature)
  init_func.info = info
  init_func._fullname = info.fullname + ".__init__"
+ if not existing_info:
+ updated = True
+ else:
+ previous_sym = info.names["__init__"].node
+ assert isinstance(previous_sym, FuncDef)
+ updated = old_type != previous_sym.arguments[1].variable.type
  info.names["__init__"] = SymbolTableNode(MDEF, init_func)
 
- if has_placeholder(old_type) or info.tuple_type and has_placeholder(info.tuple_type):
- self.api.defer(force_progress=True)
+ if has_placeholder(old_type):
+ self.api.process_placeholder(None, "NewType base", info, force_progress=updated)
  return info
 
  # Helpers

mypy/semanal_shared.py

@@ -232,6 +232,12 @@ def qualified_name(self, n: str) -> str:
  def is_typeshed_stub_file(self) -> bool:
  raise NotImplementedError
 
+ @abstractmethod
+ def process_placeholder(
+ self, name: str | None, kind: str, ctx: Context, force_progress: bool = False
+ ) -> None:
+ raise NotImplementedError
+
 
 def set_callable_name(sig: Type, fdef: FuncDef) -> ProperType:
  sig = get_proper_type(sig)

mypy/semanal_typeddict.py

@@ -535,7 +535,9 @@ def build_typeddict_typeinfo(
  info = existing_info or self.api.basic_new_typeinfo(name, fallback, line)
  typeddict_type = TypedDictType(dict(zip(items, types)), required_keys, fallback)
  if info.special_alias and has_placeholder(info.special_alias.target):
- self.api.defer(force_progress=True)
+ self.api.process_placeholder(
+ None, "TypedDict item", info, force_progress=typeddict_type != info.typeddict_type
+ )
  info.update_typeddict_type(typeddict_type)
  return info
 

mypy/types.py

@@ -2857,6 +2857,14 @@ def accept(self, visitor: TypeVisitor[T]) -> T:
  assert isinstance(visitor, SyntheticTypeVisitor)
  return cast(T, visitor.visit_placeholder_type(self))
 
+ def __hash__(self) -> int:
+ return hash((self.fullname, tuple(self.args)))
+
+ def __eq__(self, other: object) -> bool:
+ if not isinstance(other, PlaceholderType):
+ return NotImplemented
+ return self.fullname == other.fullname and self.args == other.args
+
  def serialize(self) -> str:
  # We should never get here since all placeholders should be replaced
  # during semantic analysis.

test-data/unit/fine-grained-follow-imports.test

@@ -769,3 +769,80 @@ from . import mod3
 ==
 main.py:1: error: Cannot find implementation or library stub for module named "pkg"
 main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+
+[case testNewImportCycleTypeVarBound]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+# cmd2: mypy other.py
+
+[file main.py]
+# empty
+
+[file other.py.2]
+import trio
+
+[file trio/__init__.py.2]
+from typing import TypeVar
+import trio
+from . import abc as abc
+
+T = TypeVar("T", bound=trio.abc.A)
+
+[file trio/abc.py.2]
+import trio
+class A: ...
+[out]
+==
+
+[case testNewImportCycleTupleBase]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+# cmd2: mypy other.py
+
+[file main.py]
+# empty
+
+[file other.py.2]
+import trio
+
+[file trio/__init__.py.2]
+from typing import TypeVar, Tuple
+import trio
+from . import abc as abc
+
+class C(Tuple[trio.abc.A, trio.abc.A]): ...
+
+[file trio/abc.py.2]
+import trio
+class A: ...
+[builtins fixtures/tuple.pyi]
+[out]
+==
+
+[case testNewImportCycleTypedDict]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+# cmd2: mypy other.py
+
+[file main.py]
+# empty
+
+[file other.py.2]
+import trio
+
+[file trio/__init__.py.2]
+from typing import TypeVar
+from typing_extensions import TypedDict
+import trio
+from . import abc as abc
+
+class C(TypedDict):
+ x: trio.abc.A
+ y: trio.abc.A
+
+[file trio/abc.py.2]
+import trio
+class A: ...
+[builtins fixtures/dict.pyi]
+[out]
+==