Fix sphinx.ext.autodoc.preserve_defaults (#11550)

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>

CHANGES

@@ -27,6 +27,8 @@ Deprecated
   ``sphinx.builders.html.StandaloneHTMLBuilder.script_files``.
   Use ``sphinx.application.Sphinx.add_css_file()``
   and ``sphinx.application.Sphinx.add_js_file()`` instead.
+* #11459: Deprecate ``sphinx.ext.autodoc.preserve_defaults.get_function_def()``.
+  Patch by Bénédikt Tran.
 
 Features added
 --------------
@@ -91,6 +93,9 @@ Bugs fixed
 * #11594: HTML Theme: Enhancements to horizontal scrolling on smaller
   devices in the ``agogo`` theme.
   Patch by Lukas Engelter.
+* #11459: Fix support for async and lambda functions in
+  ``sphinx.ext.autodoc.preserve_defaults``.
+  Patch by Bénédikt Tran.
 
 Testing
 -------

doc/extdev/deprecated.rst

@@ -22,6 +22,11 @@ The following is a list of deprecated interfaces.
      - Removed
      - Alternatives
 
+   * - ``sphinx.ext.autodoc.preserve_defaults.get_function_def()``
+     - 7.2
+     - 9.0
+     - N/A (replacement is private)
+
    * - ``sphinx.builders.html.StandaloneHTMLBuilder.css_files``
      - 7.2
      - 9.0

sphinx/ext/autodoc/preserve_defaults.py

@@ -8,17 +8,23 @@
 
 import ast
 import inspect
-from typing import TYPE_CHECKING, Any
+import types
+import warnings
+from typing import TYPE_CHECKING
 
 import sphinx
+from sphinx.deprecation import RemovedInSphinx90Warning
 from sphinx.locale import __
 from sphinx.pycode.ast import unparse as ast_unparse
 from sphinx.util import logging
 
 if TYPE_CHECKING:
+    from typing import Any
+
     from sphinx.application import Sphinx
 
 logger = logging.getLogger(__name__)
+_LAMBDA_NAME = (lambda: None).__name__
 
 
 class DefaultValue:
@@ -31,12 +37,19 @@ def __repr__(self) -> str:
 
 def get_function_def(obj: Any) -> ast.FunctionDef | None:
     """Get FunctionDef object from living object.
+
     This tries to parse original code for living object and returns
     AST node for given *obj*.
     """
+    warnings.warn('sphinx.ext.autodoc.preserve_defaults.get_function_def is'
+                  ' deprecated and scheduled for removal in Sphinx 9.'
+                  ' Use sphinx.ext.autodoc.preserve_defaults._get_arguments() to'
+                  ' extract AST arguments objects from a lambda or regular'
+                  ' function.', RemovedInSphinx90Warning, stacklevel=2)
+
     try:
         source = inspect.getsource(obj)
-        if source.startswith((' ', r'\t')):
+        if source.startswith((' ', '\t')):
             # subject is placed inside class or block.  To read its docstring,
             # this adds if-block before the declaration.
             module = ast.parse('if True:\n' + source)
@@ -48,6 +61,53 @@ def get_function_def(obj: Any) -> ast.FunctionDef | None:
         return None
 
 
+def _get_arguments(obj: Any, /) -> ast.arguments | None:
+    """Parse 'ast.arguments' from an object.
+
+    This tries to parse the original code for an object and returns
+    an 'ast.arguments' node.
+    """
+    try:
+        source = inspect.getsource(obj)
+        if source.startswith((' ', '\t')):
+            # 'obj' is in some indented block.
+            module = ast.parse('if True:\n' + source)
+            subject = module.body[0].body[0]  # type: ignore[attr-defined]
+        else:
+            module = ast.parse(source)
+            subject = module.body[0]
+    except (OSError, TypeError):
+        # bail; failed to load source for 'obj'.
+        return None
+    except SyntaxError:
+        if _is_lambda(obj):
+            # Most likely a multi-line arising from detecting a lambda, e.g.:
+            #
+            # class Egg:
+            #     x = property(
+            #         lambda self: 1, doc="...")
+            return None
+
+        # Other syntax errors that are not due to the fact that we are
+        # documenting a lambda function are propagated
+        # (in particular if a lambda is renamed by the user).
+        raise
+
+    return _get_arguments_inner(subject)
+
+
+def _is_lambda(x, /):
+    return isinstance(x, types.LambdaType) and x.__name__ == _LAMBDA_NAME
+
+
+def _get_arguments_inner(x: Any, /) -> ast.arguments | None:
+    if isinstance(x, (ast.AsyncFunctionDef, ast.FunctionDef, ast.Lambda)):
+        return x.args
+    if isinstance(x, (ast.Assign, ast.AnnAssign)):
+        return _get_arguments_inner(x.value)
+    return None
+
+
 def get_default_value(lines: list[str], position: ast.AST) -> str | None:
     try:
         if position.lineno == position.end_lineno:
@@ -67,18 +127,24 @@ def update_defvalue(app: Sphinx, obj: Any, bound_method: bool) -> None:
 
     try:
         lines = inspect.getsource(obj).splitlines()
-        if lines[0].startswith((' ', r'\t')):
-            lines.insert(0, '')  # insert a dummy line to follow what get_function_def() does.
+        if lines[0].startswith((' ', '\t')):
+            # insert a dummy line to follow what _get_arguments() does.
+            lines.insert(0, '')
     except (OSError, TypeError):
         lines = []
 
     try:
-        function = get_function_def(obj)
-        assert function is not None  # for mypy
-        if function.args.defaults or function.args.kw_defaults:
+        args = _get_arguments(obj)
+        if args is None:
+            # If the object is a built-in, we won't be always able to recover
+            # the function definition and its arguments. This happens if *obj*
+            # is the `__init__` method generated automatically for dataclasses.
+            return
+
+        if args.defaults or args.kw_defaults:
             sig = inspect.signature(obj)
-            defaults = list(function.args.defaults)
-            kw_defaults = list(function.args.kw_defaults)
+            defaults = list(args.defaults)
+            kw_defaults = list(args.kw_defaults)
             parameters = list(sig.parameters.values())
             for i, param in enumerate(parameters):
                 if param.default is param.empty:

tests/roots/test-ext-autodoc/target/preserve_defaults.py

@@ -30,3 +30,31 @@ def clsmeth(cls, name: str = CONSTANT, sentinel: Any = SENTINEL,
                 now: datetime = datetime.now(), color: int = 0xFFFFFF,
                 *, kwarg1, kwarg2 = 0xFFFFFF) -> None:
         """docstring"""
+
+
+get_sentinel = lambda custom=SENTINEL: custom
+"""docstring"""
+
+
+class MultiLine:
+    """docstring"""
+
+    # The properties will raise a silent SyntaxError because "lambda self: 1"
+    # will be detected as a function to update the default values of. However,
+    # only prop3 will not fail because it's on a single line whereas the others
+    # will fail to parse.
+
+    prop1 = property(
+      lambda self: 1, doc="docstring")
+
+    prop2 = property(
+      lambda self: 2, doc="docstring"
+    )
+
+    prop3 = property(lambda self: 3, doc="docstring")
+
+    prop4 = (property
+    (lambda self: 4, doc="docstring"))
+
+    prop5 = property\
+    (lambda self: 5, doc="docstring")

tests/roots/test-ext-autodoc/target/preserve_defaults_special_constructs.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from collections import namedtuple
+from dataclasses import dataclass, field
+from typing import NamedTuple, TypedDict
+
+#: docstring
+SENTINEL = object()
+
+
+#: docstring
+ze_lambda = lambda z=SENTINEL: None
+
+
+def foo(x, y, z=SENTINEL):
+    """docstring"""
+
+
+@dataclass
+class DataClass:
+    """docstring"""
+    a: int
+    b: object = SENTINEL
+    c: list[int] = field(default_factory=lambda: [1, 2, 3])
+
+
+@dataclass(init=False)
+class DataClassNoInit:
+    """docstring"""
+    a: int
+    b: object = SENTINEL
+    c: list[int] = field(default_factory=lambda: [1, 2, 3])
+
+
+class MyTypedDict(TypedDict):
+    """docstring"""
+    a: int
+    b: object
+    c: list[int]
+
+
+class MyNamedTuple1(NamedTuple):
+    """docstring"""
+    a: int
+    b: object = object()
+    c: list[int] = [1, 2, 3]
+
+
+class MyNamedTuple2(namedtuple('Base', ('a', 'b'), defaults=(0, SENTINEL))):
+    """docstring"""

tests/test_ext_autodoc_preserve_defaults.py

@@ -40,11 +40,153 @@ def test_preserve_defaults(app):
         '      docstring',
         '',
         '',
+        '.. py:class:: MultiLine()',
+        '   :module: target.preserve_defaults',
+        '',
+        '   docstring',
+        '',
+        '',
+        '   .. py:property:: MultiLine.prop1',
+        '      :module: target.preserve_defaults',
+        '',
+        '      docstring',
+        '',
+        '',
+        '   .. py:property:: MultiLine.prop2',
+        '      :module: target.preserve_defaults',
+        '',
+        '      docstring',
+        '',
+        '',
+        '   .. py:property:: MultiLine.prop3',
+        '      :module: target.preserve_defaults',
+        '',
+        '      docstring',
+        '',
+        '',
+        '   .. py:property:: MultiLine.prop4',
+        '      :module: target.preserve_defaults',
+        '',
+        '      docstring',
+        '',
+        '',
+        '   .. py:property:: MultiLine.prop5',
+        '      :module: target.preserve_defaults',
+        '',
+        '      docstring',
+        '',
+        '',
         '.. py:function:: foo(name: str = CONSTANT, sentinel: ~typing.Any = SENTINEL, '
         'now: ~datetime.datetime = datetime.now(), color: int = %s, *, kwarg1, '
         'kwarg2=%s) -> None' % (color, color),
         '   :module: target.preserve_defaults',
         '',
         '   docstring',
         '',
+        '',
+        '.. py:function:: get_sentinel(custom=SENTINEL)',
+        '   :module: target.preserve_defaults',
+        '',
+        '   docstring',
+        '',
+    ]
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc',
+                    confoverrides={'autodoc_preserve_defaults': True})
+def test_preserve_defaults_special_constructs(app):
+    options = {"members": None}
+    actual = do_autodoc(app, 'module', 'target.preserve_defaults_special_constructs', options)
+
+    # * dataclasses.dataclass:
+    #   - __init__ source code is not available
+    #   - default values specified at class level are not discovered
+    #   - values wrapped in a field(...) expression cannot be analyzed
+    #     easily even if annotations were to be parsed
+    # * typing.NamedTuple:
+    #   - __init__ source code is not available
+    #   - default values specified at class level are not discovered
+    # * collections.namedtuple:
+    #   - default values are specified as "default=(d1, d2, ...)"
+    #
+    # In the future, it might be possible to find some additional default
+    # values by parsing the source code of the annotations but the task is
+    # rather complex.
+
+    assert list(actual) == [
+        '',
+        '.. py:module:: target.preserve_defaults_special_constructs',
+        '',
+        '',
+        '.. py:class:: DataClass('
+        'a: int, b: object = <object object>, c: list[int] = <factory>)',
+        '   :module: target.preserve_defaults_special_constructs',
+        '',
+        '   docstring',
+        '',
+        '',
+        '.. py:class:: DataClassNoInit()',
+        '   :module: target.preserve_defaults_special_constructs',
+        '',
+        '   docstring',
+        '',
+        '',
+        '.. py:class:: MyNamedTuple1('
+        'a: int, b: object = <object object>, c: list[int] = [1, 2, 3])',
+        '   :module: target.preserve_defaults_special_constructs',
+        '',
+        '   docstring',
+        '',
+        '',
+        '   .. py:attribute:: MyNamedTuple1.a',
+        '      :module: target.preserve_defaults_special_constructs',
+        '      :type: int',
+        '',
+        '      Alias for field number 0',
+        '',
+        '',
+        '   .. py:attribute:: MyNamedTuple1.b',
+        '      :module: target.preserve_defaults_special_constructs',
+        '      :type: object',
+        '',
+        '      Alias for field number 1',
+        '',
+        '',
+        '   .. py:attribute:: MyNamedTuple1.c',
+        '      :module: target.preserve_defaults_special_constructs',
+        '      :type: list[int]',
+        '',
+        '      Alias for field number 2',
+        '',
+        '',
+        '.. py:class:: MyNamedTuple2(a=0, b=<object object>)',
+        '   :module: target.preserve_defaults_special_constructs',
+        '',
+        '   docstring',
+        '',
+        '',
+        '.. py:class:: MyTypedDict',
+        '   :module: target.preserve_defaults_special_constructs',
+        '',
+        '   docstring',
+        '',
+        '',
+        '.. py:data:: SENTINEL',
+        '   :module: target.preserve_defaults_special_constructs',
+        '   :value: <object object>',
+        '',
+        '   docstring',
+        '',
+        '',
+        '.. py:function:: foo(x, y, z=SENTINEL)',
+        '   :module: target.preserve_defaults_special_constructs',
+        '',
+        '   docstring',
+        '',
+        '',
+        '.. py:function:: ze_lambda(z=SENTINEL)',
+        '   :module: target.preserve_defaults_special_constructs',
+        '',
+        '   docstring',
+        '',
     ]