Improve stubgen tests (#16760)

Improve test cases around #16486

This PR does not change any actual mypy behavior, only hardens the
stubgen tests. The specific changes are:

- **dedicated test cases**: The existing pybind11 test cases originate
from a pybind11 demo. They cover a specific topic involving geometry
types and semi-implemented logic related to it. This somehow distracts
from the aspects we are trying to test here from the mypy stubgen
perspective, because it hides the actual intent of the bindings. I've
simply started adding new test cases that clearly express via their name
what the test case is addressing. I've kept the original demo stuff for
now, so that the new cases are just an addition (overhead is
negligible).
- **running mypy on the generated stubs**: In general it is crucial that
the output produced by the stubgen can actually be type checked by mypy
(this was the regression in #18486). This wasn't covered by the CI check
so far. I've added check now, which would have avoided the regression.
My goal for follow up PRs would be that we can use `mypy
--disallow-untyped-defs` or even `mypy --strict` on the output.
- **minor directory restructuring**: So far the expected stub outputs
were stored in folder names `stubgen` and `stubgen-include-docs`. This
was a bit confusing, because the folder `stubgen` suggested it contains
_the_ stubgen (implementation). I went for `expected_stubs_no_docs` and
`expected_stubs_with_docs` to make the role of the two folders more
explicit.
- **minor script bugfix**: Fix a bug in `test-stubgen.sh`: The
pre-delete functionality was broken, because the `*` was quoted and
therefore did not match.

.github/workflows/test_stubgenc.yml

@@ -1,4 +1,4 @@
-name: Test stubgenc on pybind11-mypy-demo
+name: Test stubgenc on pybind11_fixtures
 
 on:
   workflow_dispatch:

misc/test-stubgenc.sh

@@ -7,7 +7,7 @@ cd "$(dirname "$0")/.."
 
 # Install dependencies, demo project and mypy
 python -m pip install -r test-requirements.txt
-python -m pip install ./test-data/pybind11_mypy_demo
+python -m pip install ./test-data/pybind11_fixtures
 python -m pip install .
 
 EXIT=0
@@ -17,19 +17,29 @@ EXIT=0
 # everything else is passed to stubgen as its arguments
 function stubgenc_test() {
     # Remove expected stubs and generate new inplace
-    STUBGEN_OUTPUT_FOLDER=./test-data/pybind11_mypy_demo/$1
-    rm -rf "${STUBGEN_OUTPUT_FOLDER:?}/*"
+    STUBGEN_OUTPUT_FOLDER=./test-data/pybind11_fixtures/$1
+    rm -rf "${STUBGEN_OUTPUT_FOLDER:?}"
+
     stubgen -o "$STUBGEN_OUTPUT_FOLDER" "${@:2}"
 
+    # Check if generated stubs can actually be type checked by mypy
+    if ! mypy "$STUBGEN_OUTPUT_FOLDER";
+    then
+        echo "Stubgen test failed, because generated stubs failed to type check."
+        EXIT=1
+    fi
+
     # Compare generated stubs to expected ones
     if ! git diff --exit-code "$STUBGEN_OUTPUT_FOLDER";
     then
+        echo "Stubgen test failed, because generated stubs differ from expected outputs."
         EXIT=1
     fi
 }
 
 # create stubs without docstrings
-stubgenc_test stubgen -p pybind11_mypy_demo
+stubgenc_test expected_stubs_no_docs -p pybind11_fixtures
 # create stubs with docstrings
-stubgenc_test stubgen-include-docs -p pybind11_mypy_demo --include-docstrings
+stubgenc_test expected_stubs_with_docs -p pybind11_fixtures --include-docstrings
+
 exit $EXIT

test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi

@@ -0,0 +1,27 @@
+import os
+from . import demo as demo
+from typing import List, Optional, Tuple, overload
+
+class StaticMethods:
+    def __init__(self, *args, **kwargs) -> None: ...
+    @overload
+    @staticmethod
+    def overloaded_static_method(value: int) -> int: ...
+    @overload
+    @staticmethod
+    def overloaded_static_method(value: float) -> float: ...
+    @staticmethod
+    def some_static_method(a: int, b: int) -> int: ...
+
+class TestStruct:
+    field_readwrite: int
+    field_readwrite_docstring: int
+    def __init__(self, *args, **kwargs) -> None: ...
+    @property
+    def field_readonly(self) -> int: ...
+
+def func_incomplete_signature(*args, **kwargs): ...
+def func_returning_optional() -> Optional[int]: ...
+def func_returning_pair() -> Tuple[int, float]: ...
+def func_returning_path() -> os.PathLike: ...
+def func_returning_vector() -> List[float]: ...

test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/basics.pyi → test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi

@@ -3,17 +3,6 @@ from typing import ClassVar, List, overload
 PI: float
 __version__: str
 
-class Foo:
-    def __init__(self, *args, **kwargs) -> None: ...
-    @overload
-    @staticmethod
-    def overloaded_static_method(value: int) -> int: ...
-    @overload
-    @staticmethod
-    def overloaded_static_method(value: float) -> float: ...
-    @staticmethod
-    def some_static_method(a: int, b: int) -> int: ...
-
 class Point:
     class AngleUnit:
         __members__: ClassVar[dict] = ...  # read-only

test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi

@@ -0,0 +1,52 @@
+import os
+from . import demo as demo
+from typing import List, Optional, Tuple, overload
+
+class StaticMethods:
+    def __init__(self, *args, **kwargs) -> None:
+        """Initialize self.  See help(type(self)) for accurate signature."""
+    @overload
+    @staticmethod
+    def overloaded_static_method(value: int) -> int:
+        """overloaded_static_method(*args, **kwargs)
+        Overloaded function.
+
+        1. overloaded_static_method(value: int) -> int
+
+        2. overloaded_static_method(value: float) -> float
+        """
+    @overload
+    @staticmethod
+    def overloaded_static_method(value: float) -> float:
+        """overloaded_static_method(*args, **kwargs)
+        Overloaded function.
+
+        1. overloaded_static_method(value: int) -> int
+
+        2. overloaded_static_method(value: float) -> float
+        """
+    @staticmethod
+    def some_static_method(a: int, b: int) -> int:
+        """some_static_method(a: int, b: int) -> int
+
+        None
+        """
+
+class TestStruct:
+    field_readwrite: int
+    field_readwrite_docstring: int
+    def __init__(self, *args, **kwargs) -> None:
+        """Initialize self.  See help(type(self)) for accurate signature."""
+    @property
+    def field_readonly(self) -> int: ...
+
+def func_incomplete_signature(*args, **kwargs):
+    """func_incomplete_signature() -> dummy_sub_namespace::HasNoBinding"""
+def func_returning_optional() -> Optional[int]:
+    """func_returning_optional() -> Optional[int]"""
+def func_returning_pair() -> Tuple[int, float]:
+    """func_returning_pair() -> Tuple[int, float]"""
+def func_returning_path() -> os.PathLike:
+    """func_returning_path() -> os.PathLike"""
+def func_returning_vector() -> List[float]:
+    """func_returning_vector() -> List[float]"""

test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/basics.pyi → test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi

@@ -3,52 +3,22 @@ from typing import ClassVar, List, overload
 PI: float
 __version__: str
 
-class Foo:
-    def __init__(self, *args, **kwargs) -> None:
-        """Initialize self.  See help(type(self)) for accurate signature."""
-    @overload
-    @staticmethod
-    def overloaded_static_method(value: int) -> int:
-        """overloaded_static_method(*args, **kwargs)
-        Overloaded function.
-
-        1. overloaded_static_method(value: int) -> int
-
-        2. overloaded_static_method(value: float) -> float
-        """
-    @overload
-    @staticmethod
-    def overloaded_static_method(value: float) -> float:
-        """overloaded_static_method(*args, **kwargs)
-        Overloaded function.
-
-        1. overloaded_static_method(value: int) -> int
-
-        2. overloaded_static_method(value: float) -> float
-        """
-    @staticmethod
-    def some_static_method(a: int, b: int) -> int:
-        """some_static_method(a: int, b: int) -> int
-
-        None
-        """
-
 class Point:
     class AngleUnit:
         __members__: ClassVar[dict] = ...  # read-only
         __entries: ClassVar[dict] = ...
         degree: ClassVar[Point.AngleUnit] = ...
         radian: ClassVar[Point.AngleUnit] = ...
         def __init__(self, value: int) -> None:
-            """__init__(self: pybind11_mypy_demo.basics.Point.AngleUnit, value: int) -> None"""
+            """__init__(self: pybind11_fixtures.demo.Point.AngleUnit, value: int) -> None"""
         def __eq__(self, other: object) -> bool:
             """__eq__(self: object, other: object) -> bool"""
         def __hash__(self) -> int:
             """__hash__(self: object) -> int"""
         def __index__(self) -> int:
-            """__index__(self: pybind11_mypy_demo.basics.Point.AngleUnit) -> int"""
+            """__index__(self: pybind11_fixtures.demo.Point.AngleUnit) -> int"""
         def __int__(self) -> int:
-            """__int__(self: pybind11_mypy_demo.basics.Point.AngleUnit) -> int"""
+            """__int__(self: pybind11_fixtures.demo.Point.AngleUnit) -> int"""
         def __ne__(self, other: object) -> bool:
             """__ne__(self: object, other: object) -> bool"""
         @property
@@ -63,15 +33,15 @@ class Point:
         mm: ClassVar[Point.LengthUnit] = ...
         pixel: ClassVar[Point.LengthUnit] = ...
         def __init__(self, value: int) -> None:
-            """__init__(self: pybind11_mypy_demo.basics.Point.LengthUnit, value: int) -> None"""
+            """__init__(self: pybind11_fixtures.demo.Point.LengthUnit, value: int) -> None"""
         def __eq__(self, other: object) -> bool:
             """__eq__(self: object, other: object) -> bool"""
         def __hash__(self) -> int:
             """__hash__(self: object) -> int"""
         def __index__(self) -> int:
-            """__index__(self: pybind11_mypy_demo.basics.Point.LengthUnit) -> int"""
+            """__index__(self: pybind11_fixtures.demo.Point.LengthUnit) -> int"""
         def __int__(self) -> int:
-            """__int__(self: pybind11_mypy_demo.basics.Point.LengthUnit) -> int"""
+            """__int__(self: pybind11_fixtures.demo.Point.LengthUnit) -> int"""
         def __ne__(self, other: object) -> bool:
             """__ne__(self: object, other: object) -> bool"""
         @property
@@ -90,38 +60,38 @@ class Point:
         """__init__(*args, **kwargs)
         Overloaded function.
 
-        1. __init__(self: pybind11_mypy_demo.basics.Point) -> None
+        1. __init__(self: pybind11_fixtures.demo.Point) -> None
 
-        2. __init__(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> None
+        2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None
         """
     @overload
     def __init__(self, x: float, y: float) -> None:
         """__init__(*args, **kwargs)
         Overloaded function.
 
-        1. __init__(self: pybind11_mypy_demo.basics.Point) -> None
+        1. __init__(self: pybind11_fixtures.demo.Point) -> None
 
-        2. __init__(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> None
+        2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None
         """
     def as_list(self) -> List[float]:
-        """as_list(self: pybind11_mypy_demo.basics.Point) -> List[float]"""
+        """as_list(self: pybind11_fixtures.demo.Point) -> List[float]"""
     @overload
     def distance_to(self, x: float, y: float) -> float:
         """distance_to(*args, **kwargs)
         Overloaded function.
 
-        1. distance_to(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> float
+        1. distance_to(self: pybind11_fixtures.demo.Point, x: float, y: float) -> float
 
-        2. distance_to(self: pybind11_mypy_demo.basics.Point, other: pybind11_mypy_demo.basics.Point) -> float
+        2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float
         """
     @overload
     def distance_to(self, other: Point) -> float:
         """distance_to(*args, **kwargs)
         Overloaded function.
 
-        1. distance_to(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> float
+        1. distance_to(self: pybind11_fixtures.demo.Point, x: float, y: float) -> float
 
-        2. distance_to(self: pybind11_mypy_demo.basics.Point, other: pybind11_mypy_demo.basics.Point) -> float
+        2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float
         """
     @property
     def length(self) -> float: ...

test-data/pybind11_mypy_demo/setup.py → test-data/pybind11_fixtures/setup.py

@@ -5,14 +5,14 @@
 # Documentation: https://pybind11.readthedocs.io/en/stable/compiling.html
 ext_modules = [
     Pybind11Extension(
-        "pybind11_mypy_demo",
+        "pybind11_fixtures",
         ["src/main.cpp"],
         cxx_std=17,
     ),
 ]
 
 setup(
-    name="pybind11-mypy-demo",
+    name="pybind11_fixtures",
     version="0.0.1",
     ext_modules=ext_modules,
 )