From c3d0a79176f1df4770172cfebe2f40b39169909d Mon Sep 17 00:00:00 2001 From: Lucas Moura Belo Date: Mon, 11 Nov 2024 08:35:52 -0300 Subject: [PATCH] Docstrings for classes and properties for extension modules (#2) * Docstrings for class properties * Docstrings for classes * Validates the include-docstrings arg to generate docstrings * Fixing test template --- mypy/stubdoc.py | 90 +++++++++++++++++++ mypy/stubgenc.py | 85 +++++++++++------- .../pybind11_fixtures/__init__.pyi | 9 +- .../pybind11_fixtures/demo.pyi | 47 ++++++++-- 4 files changed, 192 insertions(+), 39 deletions(-) diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py index 434de0ea3bcb..e030cb0a5bb3 100644 --- a/mypy/stubdoc.py +++ b/mypy/stubdoc.py @@ -72,6 +72,63 @@ def __eq__(self, other: Any) -> bool: return False +class ClassSig(NamedTuple): + name: str + base_types: list[str] | None = None + + def format_sig( + self, + indent: str, + types: list[str], + methods: list[str], + static_properties: list[str], + rw_properties: list[str], + ro_properties: list[str], + docstring: str | None = None, + ) -> list[str]: + + output: list[str] = [] + + if self.base_types: + bases_str = "(%s)" % ", ".join(self.base_types) + else: + bases_str = "" + + if docstring: + sufix = f"\n{indent} {mypy.util.quote_docstring(docstring)}\n" + else: + sufix = "" + + if types or static_properties or rw_properties or methods or ro_properties: + sig = f"{indent}class {self.name}{bases_str}:" + output.append(f"{sig}{sufix}") + + for line in types: + if ( + output + and output[-1] + and not output[-1].strip().startswith("class") + and line.strip().startswith("class") + ): + output.append("") + + output.append(line) + + for line in static_properties: + output.append(line) + for line in rw_properties: + output.append(line) + for line in methods: + output.append(line) + for line in ro_properties: + output.append(line) + else: + sig = f"{indent}class {self.name}{bases_str}: ..." + output.append(f"{sig}{sufix}") + + return output + + class FunctionSig(NamedTuple): name: str args: list[ArgSig] @@ -150,6 +207,39 @@ def format_sig( return f"{sig}{suffix}" +class PropertySig(NamedTuple): + name: str + prop_type: str + + def format_sig( + self, + indent: str = "", + is_readonly: bool | None = False, + is_static: bool | None = False, + name_ref: str | None = None, + docstring: str | None = None, + ) -> str: + + if is_static: + if docstring: + sufix = f"\n{indent}{mypy.util.quote_docstring(docstring)}" + else: + sufix = "" + + trailing_comment = " # read-only" if is_readonly else "" + sig = f"{indent}{self.name}: {name_ref}[{self.prop_type}] = ...{trailing_comment}" + + return f"{sig}{sufix}" + else: + sig = f"{indent}{self.name}: {self.prop_type}" + if docstring: + sufix = f"\n{indent}{mypy.util.quote_docstring(docstring)}" + else: + sufix = "" + + return f"{sig}{sufix}" + + # States of the docstring parser. STATE_INIT: Final = 1 STATE_FUNCTION_NAME: Final = 2 diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 1cd709b9d603..ccafe419cd13 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -18,7 +18,9 @@ from mypy.moduleinspect import is_c_module from mypy.stubdoc import ( ArgSig, + ClassSig, FunctionSig, + PropertySig, Sig, find_unique_signatures, infer_arg_sig_from_anon_docstring, @@ -649,16 +651,23 @@ def generate_function_stub( output.extend(self.format_func_def(inferred, decorators=decorators, docstring=docstring)) self._fix_iter(ctx, inferred, output) - def _indent_docstring(self, docstring: str) -> str: + def _indent_docstring( + self, docstring: str, extra_indent: bool = True, trailing_newline: bool = False + ) -> str: """Fix indentation of docstring extracted from pybind11 or other binding generators.""" lines = docstring.splitlines(keepends=True) - indent = self._indent + " " + indent = self._indent + (" " if extra_indent else "") if len(lines) > 1: if not all(line.startswith(indent) or not line.strip() for line in lines): # if the docstring is not indented, then indent all but the first line for i, line in enumerate(lines[1:]): if line.strip(): - lines[i + 1] = indent + line + # ignore any left space to keep the standard ident + lines[i + 1] = indent + line.lstrip() + + if trailing_newline and not lines[-1].endswith("\n"): + lines[-1] += "\n" + # if there's a trailing newline, add a final line to visually indent the quoted docstring if lines[-1].endswith("\n"): if len(lines) > 1: @@ -728,6 +737,13 @@ def generate_property_stub( self.record_name(ctx.name) static = self.is_static_property(raw_obj) readonly = self.is_property_readonly(raw_obj) + + if docstring: + # fields must define its docstring using the same ident + # readonly properties generates a function, + # which requires an extra ident in the first line + docstring = self._indent_docstring(docstring, extra_indent=readonly) + if static: ret_type: str | None = self.strip_or_import(self.get_type_annotation(obj)) else: @@ -738,25 +754,35 @@ def generate_property_stub( if inferred_type is not None: inferred_type = self.strip_or_import(inferred_type) + if not self._include_docstrings: + docstring = None + if static: classvar = self.add_name("typing.ClassVar") - trailing_comment = " # read-only" if readonly else "" if inferred_type is None: inferred_type = self.add_name("_typeshed.Incomplete") + prop_sig = PropertySig(name, inferred_type) static_properties.append( - f"{self._indent}{name}: {classvar}[{inferred_type}] = ...{trailing_comment}" + prop_sig.format_sig( + indent=self._indent, + is_readonly=readonly, + is_static=True, + name_ref=classvar, + docstring=docstring, + ) ) else: # regular property if readonly: ro_properties.append(f"{self._indent}@property") - sig = FunctionSig(name, [ArgSig("self")], inferred_type) - ro_properties.append(sig.format_sig(indent=self._indent)) + func_sig = FunctionSig(name, [ArgSig("self")], inferred_type) + ro_properties.append(func_sig.format_sig(indent=self._indent, docstring=docstring)) else: if inferred_type is None: inferred_type = self.add_name("_typeshed.Incomplete") - rw_properties.append(f"{self._indent}{name}: {inferred_type}") + prop_sig = PropertySig(name, inferred_type) + rw_properties.append(prop_sig.format_sig(indent=self._indent, docstring=docstring)) def get_type_fullname(self, typ: type) -> str: """Given a type, return a string representation""" @@ -859,34 +885,27 @@ def generate_class_stub( classvar = self.add_name("typing.ClassVar") static_properties.append(f"{self._indent}{attr}: {classvar}[{prop_type_name}] = ...") + docstring = class_info.docstring if self._include_docstrings else None + if docstring: + docstring = self._indent_docstring( + docstring, extra_indent=False, trailing_newline=True + ) + self.dedent() bases = self.get_base_types(cls) - if bases: - bases_str = "(%s)" % ", ".join(bases) - else: - bases_str = "" - if types or static_properties or rw_properties or methods or ro_properties: - output.append(f"{self._indent}class {class_name}{bases_str}:") - for line in types: - if ( - output - and output[-1] - and not output[-1].strip().startswith("class") - and line.strip().startswith("class") - ): - output.append("") - output.append(line) - for line in static_properties: - output.append(line) - for line in rw_properties: - output.append(line) - for line in methods: - output.append(line) - for line in ro_properties: - output.append(line) - else: - output.append(f"{self._indent}class {class_name}{bases_str}: ...") + sig = ClassSig(class_name, bases) + output.extend( + sig.format_sig( + indent=self._indent, + types=types, + methods=methods, + static_properties=static_properties, + rw_properties=rw_properties, + ro_properties=ro_properties, + docstring=docstring, + ) + ) def generate_variable_stub(self, name: str, obj: object, output: list[str]) -> None: """Generate stub for a single variable using runtime introspection. diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi index db04bccab028..10b72c3c86ca 100644 --- a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi @@ -34,11 +34,18 @@ class StaticMethods: class TestStruct: field_readwrite: int + """(self: pybind11_fixtures.TestStruct) -> int""" field_readwrite_docstring: int + """some docstring + (self: pybind11_fixtures.TestStruct) -> int + """ def __init__(self, *args, **kwargs) -> None: """Initialize self. See help(type(self)) for accurate signature.""" @property - def field_readonly(self) -> int: ... + def field_readonly(self) -> int: + """some docstring + (arg0: pybind11_fixtures.TestStruct) -> int + """ def func_incomplete_signature(*args, **kwargs): """func_incomplete_signature() -> dummy_sub_namespace::HasNoBinding""" diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi index 1be0bc905a43..472414dfc148 100644 --- a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi @@ -5,7 +5,15 @@ __version__: str class Point: class AngleUnit: + """Members: + + radian + + degree + """ + __members__: ClassVar[dict] = ... # read-only + """__members__(arg0: handle) -> dict""" __entries: ClassVar[dict] = ... degree: ClassVar[Point.AngleUnit] = ... radian: ClassVar[Point.AngleUnit] = ... @@ -22,12 +30,27 @@ class Point: def __ne__(self, other: object) -> bool: """__ne__(self: object, other: object) -> bool""" @property - def name(self) -> str: ... + def name(self) -> str: + """name(self: handle) -> str + + name(self: handle) -> str + """ @property - def value(self) -> int: ... + def value(self) -> int: + """(arg0: pybind11_fixtures.demo.Point.AngleUnit) -> int""" class LengthUnit: + """Members: + + mm + + pixel + + inch + """ + __members__: ClassVar[dict] = ... # read-only + """__members__(arg0: handle) -> dict""" __entries: ClassVar[dict] = ... inch: ClassVar[Point.LengthUnit] = ... mm: ClassVar[Point.LengthUnit] = ... @@ -45,16 +68,29 @@ class Point: def __ne__(self, other: object) -> bool: """__ne__(self: object, other: object) -> bool""" @property - def name(self) -> str: ... + def name(self) -> str: + """name(self: handle) -> str + + name(self: handle) -> str + """ @property - def value(self) -> int: ... + def value(self) -> int: + """(arg0: pybind11_fixtures.demo.Point.LengthUnit) -> int""" angle_unit: ClassVar[Point.AngleUnit] = ... + """(arg0: object) -> pybind11_fixtures.demo.Point.AngleUnit""" length_unit: ClassVar[Point.LengthUnit] = ... + """(arg0: object) -> pybind11_fixtures.demo.Point.LengthUnit""" x_axis: ClassVar[Point] = ... # read-only + """(arg0: object) -> pybind11_fixtures.demo.Point""" y_axis: ClassVar[Point] = ... # read-only + """(arg0: object) -> pybind11_fixtures.demo.Point""" origin: ClassVar[Point] = ... x: float + """some docstring + (self: pybind11_fixtures.demo.Point) -> float + """ y: float + """(arg0: pybind11_fixtures.demo.Point) -> float""" @overload def __init__(self) -> None: """__init__(*args, **kwargs) @@ -94,7 +130,8 @@ class Point: 2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float """ @property - def length(self) -> float: ... + def length(self) -> float: + """(arg0: pybind11_fixtures.demo.Point) -> float""" def answer() -> int: '''answer() -> int