stubtest: catch all for errors from the runtime (#13160)

Fixes #12931

mypy/stubtest.py

@@ -13,6 +13,7 @@
 import pkgutil
 import re
 import sys
+import traceback
 import types
 import typing
 import typing_extensions
@@ -201,7 +202,28 @@ def test_module(module_name: str) -> Iterator[Error]:
 
  with warnings.catch_warnings():
  warnings.simplefilter("ignore")
- yield from verify(stub, runtime, [module_name])
+ try:
+ yield from verify(stub, runtime, [module_name])
+ except Exception as e:
+ bottom_frame = list(traceback.walk_tb(e.__traceback__))[-1][0]
+ bottom_module = bottom_frame.f_globals.get("__name__", "")
+ # Pass on any errors originating from stubtest or mypy
+ # These can occur expectedly, e.g. StubtestFailure
+ if bottom_module == "__main__" or bottom_module.split(".")[0] == "mypy":
+ raise
+ yield Error(
+ [module_name],
+ f"encountered unexpected error, {type(e).__name__}: {e}",
+ stub,
+ runtime,
+ stub_desc="N/A",
+ runtime_desc=(
+ "This is most likely the fault of something very dynamic in your library. "
+ "It's also possible this is a bug in stubtest.\nIf in doubt, please "
+ "open an issue at https://github.com/python/mypy\n\n"
+ + traceback.format_exc().strip()
+ ),
+ )
 
 
 @singledispatch