Fix and optimise overload compatibility checking (#14018)

Discovered as part of #14017

mypy/subtypes.py

@@ -823,9 +823,8 @@ def visit_overloaded(self, left: Overloaded) -> bool:
             # Ensure each overload in the right side (the supertype) is accounted for.
             previous_match_left_index = -1
             matched_overloads = set()
-            possible_invalid_overloads = set()
 
-            for right_index, right_item in enumerate(right.items):
+            for right_item in right.items:
                 found_match = False
 
                 for left_index, left_item in enumerate(left.items):
@@ -834,43 +833,36 @@ def visit_overloaded(self, left: Overloaded) -> bool:
                     # Order matters: we need to make sure that the index of
                     # this item is at least the index of the previous one.
                     if subtype_match and previous_match_left_index <= left_index:
-                        if not found_match:
-                            # Update the index of the previous match.
-                            previous_match_left_index = left_index
-                            found_match = True
-                            matched_overloads.add(left_item)
-                            possible_invalid_overloads.discard(left_item)
+                        previous_match_left_index = left_index
+                        found_match = True
+                        matched_overloads.add(left_index)
+                        break
                     else:
                         # If this one overlaps with the supertype in any way, but it wasn't
                         # an exact match, then it's a potential error.
                         strict_concat = self.options.strict_concatenate if self.options else True
-                        if is_callable_compatible(
-                            left_item,
-                            right_item,
-                            is_compat=self._is_subtype,
-                            ignore_return=True,
-                            ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names,
-                            strict_concatenate=strict_concat,
-                        ) or is_callable_compatible(
-                            right_item,
-                            left_item,
-                            is_compat=self._is_subtype,
-                            ignore_return=True,
-                            ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names,
-                            strict_concatenate=strict_concat,
+                        if left_index not in matched_overloads and (
+                            is_callable_compatible(
+                                left_item,
+                                right_item,
+                                is_compat=self._is_subtype,
+                                ignore_return=True,
+                                ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names,
+                                strict_concatenate=strict_concat,
+                            )
+                            or is_callable_compatible(
+                                right_item,
+                                left_item,
+                                is_compat=self._is_subtype,
+                                ignore_return=True,
+                                ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names,
+                                strict_concatenate=strict_concat,
+                            )
                         ):
-                            # If this is an overload that's already been matched, there's no
-                            # problem.
-                            if left_item not in matched_overloads:
-                                possible_invalid_overloads.add(left_item)
+                            return False
 
                 if not found_match:
                     return False
-
-            if possible_invalid_overloads:
-                # There were potentially invalid overloads that were never matched to the
-                # supertype.
-                return False
             return True
         elif isinstance(right, UnboundType):
             return True

test-data/unit/check-classes.test

@@ -3872,28 +3872,59 @@ class Super:
     def foo(self, a: C) -> C: pass
 
 class Sub(Super):
-    @overload  # Fail
+    @overload
     def foo(self, a: A) -> A: pass
     @overload
     def foo(self, a: B) -> C: pass  # Fail
     @overload
     def foo(self, a: C) -> C: pass
+
+class Sub2(Super):
+    @overload
+    def foo(self, a: B) -> C: pass  # Fail
+    @overload
+    def foo(self, a: A) -> A: pass
+    @overload
+    def foo(self, a: C) -> C: pass
+
+class Sub3(Super):
+    @overload
+    def foo(self, a: A) -> int: pass
+    @overload
+    def foo(self, a: A) -> A: pass
+    @overload
+    def foo(self, a: C) -> C: pass
 [builtins fixtures/classmethod.pyi]
 [out]
-tmp/foo.pyi:16: error: Signature of "foo" incompatible with supertype "Super"
-tmp/foo.pyi:16: note:      Superclass:
-tmp/foo.pyi:16: note:          @overload
-tmp/foo.pyi:16: note:          def foo(self, a: A) -> A
-tmp/foo.pyi:16: note:          @overload
-tmp/foo.pyi:16: note:          def foo(self, a: C) -> C
-tmp/foo.pyi:16: note:      Subclass:
-tmp/foo.pyi:16: note:          @overload
-tmp/foo.pyi:16: note:          def foo(self, a: A) -> A
-tmp/foo.pyi:16: note:          @overload
-tmp/foo.pyi:16: note:          def foo(self, a: B) -> C
-tmp/foo.pyi:16: note:          @overload
-tmp/foo.pyi:16: note:          def foo(self, a: C) -> C
 tmp/foo.pyi:19: error: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader
+tmp/foo.pyi:24: error: Signature of "foo" incompatible with supertype "Super"
+tmp/foo.pyi:24: note:      Superclass:
+tmp/foo.pyi:24: note:          @overload
+tmp/foo.pyi:24: note:          def foo(self, a: A) -> A
+tmp/foo.pyi:24: note:          @overload
+tmp/foo.pyi:24: note:          def foo(self, a: C) -> C
+tmp/foo.pyi:24: note:      Subclass:
+tmp/foo.pyi:24: note:          @overload
+tmp/foo.pyi:24: note:          def foo(self, a: B) -> C
+tmp/foo.pyi:24: note:          @overload
+tmp/foo.pyi:24: note:          def foo(self, a: A) -> A
+tmp/foo.pyi:24: note:          @overload
+tmp/foo.pyi:24: note:          def foo(self, a: C) -> C
+tmp/foo.pyi:25: error: Overloaded function signatures 1 and 2 overlap with incompatible return types
+tmp/foo.pyi:32: error: Signature of "foo" incompatible with supertype "Super"
+tmp/foo.pyi:32: note:      Superclass:
+tmp/foo.pyi:32: note:          @overload
+tmp/foo.pyi:32: note:          def foo(self, a: A) -> A
+tmp/foo.pyi:32: note:          @overload
+tmp/foo.pyi:32: note:          def foo(self, a: C) -> C
+tmp/foo.pyi:32: note:      Subclass:
+tmp/foo.pyi:32: note:          @overload
+tmp/foo.pyi:32: note:          def foo(self, a: A) -> int
+tmp/foo.pyi:32: note:          @overload
+tmp/foo.pyi:32: note:          def foo(self, a: A) -> A
+tmp/foo.pyi:32: note:          @overload
+tmp/foo.pyi:32: note:          def foo(self, a: C) -> C
+tmp/foo.pyi:35: error: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader
 
 [case testTypeTypeOverlapsWithObjectAndType]
 from foo import *

test-data/unit/check-selftype.test

@@ -128,6 +128,34 @@ reveal_type(cast(A, C()).copy())  # N: Revealed type is "__main__.A"
 
 [builtins fixtures/bool.pyi]
 
+[case testSelfTypeOverrideCompatibility]
+from typing import overload, TypeVar, Generic
+
+T = TypeVar("T")
+
+class A(Generic[T]):
+    @overload
+    def f(self: A[int]) -> int: ...
+    @overload
+    def f(self: A[str]) -> str: ...
+    def f(self): ...
+
+class B(A[T]):
+    @overload
+    def f(self: A[int]) -> int: ...
+    @overload
+    def f(self: A[str]) -> str: ...
+    def f(self): ...
+
+class B2(A[T]):
+    @overload
+    def f(self: A[int]) -> int: ...
+    @overload
+    def f(self: A[str]) -> str: ...
+    @overload
+    def f(self: A[bytes]) -> bytes: ...
+    def f(self): ...
+
 [case testSelfTypeSuper]
 from typing import TypeVar, cast