stubtest: error if a method or class is abstract at runtime, but not …

…in the stub

mypy/stubtest.py

@@ -269,6 +269,20 @@ def verify_typeinfo(
     if not isinstance(runtime, type):
         yield Error(object_path, "is not a type", stub, runtime, stub_desc=repr(stub))
         return
+    if inspect.isabstract(runtime) and not stub.is_abstract:
+        abstract_attrs = [
+            k
+            for k, v in runtime.__dict__.items()
+            if is_abstractmethod(v)
+        ]
+        yield Error(
+            object_path,
+            "has abstract attributes at runtime but not in the stub",
+            stub,
+            runtime,
+            stub_desc=f"{stub!r}, with no abstract attributes",
+            runtime_desc=f"Abstract attributes are: {abstract_attrs}"
+        )
 
     try:
         class SubClass(runtime):  # type: ignore
@@ -716,6 +730,16 @@ def verify_funcitem(
         if not callable(runtime):
             return
 
+    if isinstance(stub, nodes.FuncDef) and not stub.is_abstract and is_abstractmethod(runtime):
+        yield Error(
+            object_path,
+            "is abstract at runtime, but not in the stub",
+            stub,
+            runtime,
+            stub_desc="A concrete method",
+            runtime_desc="An abstract method"
+        )
+
     for message in _verify_static_class_methods(stub, runtime, object_path):
         yield Error(object_path, "is inconsistent, " + message, stub, runtime)
 
@@ -1031,6 +1055,10 @@ def verify_typealias(
 )
 
 
+def is_abstractmethod(runtime: object) -> bool:
+    return getattr(runtime, "__isabstractmethod__", False)
+
+
 def is_probably_private(name: str) -> bool:
     return name.startswith("_") and not is_dunder(name)
 

mypy/test/teststubtest.py

@@ -125,7 +125,7 @@ class Case:
     def __init__(self, stub: str, runtime: str, error: Optional[str]):
         self.stub = stub
         self.runtime = runtime
-        self.error = error
+        self.errors: Optional[List[str]] = None if error is None else error.split(', ')
 
 
 def collect_cases(fn: Callable[..., Iterator[Case]]) -> Callable[..., None]:
@@ -140,18 +140,18 @@ def test(*args: Any, **kwargs: Any) -> None:
         cases = list(fn(*args, **kwargs))
         expected_errors = set()
         for c in cases:
-            if c.error is None:
+            if c.errors is None:
                 continue
-            expected_error = c.error
-            if expected_error == "":
-                expected_error = TEST_MODULE_NAME
-            elif not expected_error.startswith(f"{TEST_MODULE_NAME}."):
-                expected_error = f"{TEST_MODULE_NAME}.{expected_error}"
-            assert expected_error not in expected_errors, (
-                "collect_cases merges cases into a single stubtest invocation; we already "
-                "expect an error for {}".format(expected_error)
-            )
-            expected_errors.add(expected_error)
+            for expected_error in c.errors:
+                if expected_error == "":
+                    expected_error = TEST_MODULE_NAME
+                elif not expected_error.startswith(f"{TEST_MODULE_NAME}."):
+                    expected_error = f"{TEST_MODULE_NAME}.{expected_error}"
+                assert expected_error not in expected_errors, (
+                    "collect_cases merges cases into a single stubtest invocation; we already "
+                    "expect an error for {}".format(expected_error)
+                )
+                expected_errors.add(expected_error)
         output = run_stubtest(
             stub="\n\n".join(textwrap.dedent(c.stub.lstrip("\n")) for c in cases),
             runtime="\n\n".join(textwrap.dedent(c.runtime.lstrip("\n")) for c in cases),
@@ -230,6 +230,52 @@ def test_coroutines(self) -> Iterator[Case]:
             error=None,
         )
 
+    @collect_cases
+    def test_abstract_classes(self) -> Iterator[Case]:
+        yield Case(
+            stub="""
+            class X:
+                def bar(self) -> None: ...
+            """,
+            runtime="""
+            from abc import abstractmethod
+            class X:
+                @abstractmethod
+                def bar(self): raise NotImplementedError
+            """,
+            # runtime method is abstract, stub class isn't
+            # neither stub nor runtime class is abstract
+            error="X.bar",
+        )
+        yield Case(
+            stub="""
+            class Y:
+                def bar(self) -> None: ...
+            """,
+            runtime="""
+            from abc import ABC, abstractmethod
+            class Y(ABC):
+                @abstractmethod
+                def bar(self): raise NotImplementedError
+            """,
+            error="Y, Y.bar",  # runtime method and class are both abstract, stub isn't for either
+        )
+        yield Case(
+            stub="""
+            from abc import abstractmethod
+            class Z:
+                @abstractmethod
+                def bar(self) -> None: ...
+            """,
+            runtime="""
+            from abc import ABC, abstractmethod
+            class Z(ABC):
+                @abstractmethod
+                def bar(self): raise NotImplementedError
+            """,
+            error=None,
+        )
+
     @collect_cases
     def test_arg_name(self) -> Iterator[Case]:
         yield Case(