Fix docstring parsing (#953)

The fallback option in `iminuit.util.describe` to the docstring failed
on modern docstrings with Python type annotations `foo(x: MyType, ...)`.

This patch fixes that.

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Henry Schreiner <henry.fredrick.schreiner@cern.ch>

src/iminuit/util.py

@@ -1136,7 +1136,7 @@ def describe(callable, *, annotations=False):
     Parameters
     ----------
     callable : callable
-        Callable whose parameters should be extracted.
+        Callable whose parameter names should be extracted.
     annotations : bool, optional
         Whether to also extract annotations. Default is false.
 
@@ -1153,7 +1153,7 @@ def describe(callable, *, annotations=False):
     Parameter names are extracted with the following three methods, which are attempted
     in order. The first to succeed determines the result.
 
-    1.  Using ``obj._parameters``, which is a dict that makes parameter names to
+    1.  Using ``obj._parameters``, which is a dict that maps parameter names to
         parameter limits or None if the parameter has to limits. Users are encouraged
         to use this mechanism to provide signatures for objects that otherwise would
         not have a detectable signature. Example::
@@ -1208,28 +1208,28 @@ def my_cost_function(a: float, b: Annotated[float, 0:]):
 
     3.  Using the docstring. The docstring is textually parsed to detect the parameter
         names. This requires that a docstring is present which follows the Python
-        standard formatting for function signatures.
+        standard formatting for function signatures. Here is a contrived example which
+        is parsed correctly: fn(a, b: float, int c, d=1, e: float=2.2, double e=3).
 
     Ambiguous cases with positional and keyword argument are handled in the following
     way::
 
+        def fcn(a, b, *args, **kwargs): ...
         # describe returns [a, b];
         # *args and **kwargs are ignored
-        def fcn(a, b, *args, **kwargs): ...
 
+        def fcn(a, b, c=1): ...
         # describe returns [a, b, c];
         # positional arguments with default values are detected
-        def fcn(a, b, c=1): ...
-
     """
     if _address_of_cfunc(callable) != 0:
         return {} if annotations else []
 
     args = (
         getattr(callable, "_parameters", {})
-        or _arguments_from_func_code(callable)
-        or _parameters_from_inspect(callable)
-        or _arguments_from_docstring(callable)
+        or _describe_impl_func_code(callable)
+        or _describe_impl_inspect(callable)
+        or _describe_impl_docstring(callable)
     )
 
     if annotations:
@@ -1238,7 +1238,7 @@ def fcn(a, b, c=1): ...
     return list(args)
 
 
-def _arguments_from_func_code(callable):
+def _describe_impl_func_code(callable):
     # Check (faked) f.func_code; for backward-compatibility with iminuit-1.x
     if hasattr(callable, "func_code"):
         # cannot warn about deprecation here, since numba.njit also uses .func_code
@@ -1247,57 +1247,7 @@ def _arguments_from_func_code(callable):
     return {}
 
 
-def _get_limit(annotation: Union[type, Annotated[float, Any], str]):
-    from iminuit import typing
-
-    if isinstance(annotation, str):
-        # This provides limited support for string annotations, which
-        # have a lot of problems, see https://peps.python.org/pep-0649.
-        try:
-            annotation = eval(annotation, None, typing.__dict__)
-        except NameError:
-            # We ignore unknown annotations to fix issue #846.
-            # I cannot replicate here what inspect.signature(..., eval_str=True) does.
-            # I need a dict with the global objects at the call site of describe, but
-            # it is not globals(). Anyway, when using strings, only the annotations
-            # from the package annotated-types are supported.
-            pass
-
-    if annotation == inspect.Parameter.empty:
-        return None
-
-    if get_origin(annotation) is not Annotated:
-        return None
-
-    tp, *constraints = get_args(annotation)
-    assert tp is float
-    lim = [-np.inf, np.inf]
-    for c in constraints:
-        if isinstance(c, slice):
-            if c.start is not None:
-                lim[0] = c.start
-            if c.stop is not None:
-                lim[1] = c.stop
-        # Minuit does not distinguish between closed and open intervals.
-        # We use a chain of ifs so that the code also works with the
-        # `Interval` class, which contains several of those attributes
-        # and which can be None.
-        gt = getattr(c, "gt", None)
-        ge = getattr(c, "ge", None)
-        lt = getattr(c, "lt", None)
-        le = getattr(c, "le", None)
-        if gt is not None:
-            lim[0] = gt
-        if ge is not None:
-            lim[0] = ge
-        if lt is not None:
-            lim[1] = lt
-        if le is not None:
-            lim[1] = le
-    return tuple(lim)
-
-
-def _parameters_from_inspect(callable):
+def _describe_impl_inspect(callable):
     try:
         signature = inspect.signature(callable)
     except ValueError:  # raised when used on built-in function
@@ -1315,7 +1265,7 @@ def _parameters_from_inspect(callable):
     return r
 
 
-def _arguments_from_docstring(callable):
+def _describe_impl_docstring(callable):
     doc = inspect.getdoc(callable)
 
     if doc is None:
@@ -1325,6 +1275,7 @@ def _arguments_from_docstring(callable):
     #   min(iterable, *[, default=obj, key=func]) -> value
     #   min(arg1, arg2, *args, *[, key=func]) -> value
     #   Foo.bar(self, int ncall_me =10000, [resume=True, int nsplit=1])
+    #   Foo.baz(self: Foo, ncall_me: int =10000)
 
     try:
         # function wrapper functools.partial does not offer __name__,
@@ -1350,12 +1301,13 @@ def _arguments_from_docstring(callable):
     items = [x.strip(" []") for x in doc[start : start + ich].split(",")]
 
     # strip self if callable is a class method
-    if items[0] == "self":
+    if inspect.ismethod(callable):
         items = items[1:]
 
     #   "iterable", "*", "default=obj", "key=func"
     #   "arg1", "arg2", "*args", "*", "key=func"
     #   "int ncall_me =10000", "resume=True", "int nsplit=1"
+    #   "ncall_me: int =10000"
 
     try:
         i = items.index("*args")
@@ -1366,22 +1318,78 @@ def _arguments_from_docstring(callable):
     #   "iterable", "*", "default=obj", "key=func"
     #   "arg1", "arg2", "*", "key=func"
     #   "int ncall_me =10000", "resume=True", "int nsplit=1"
+    #   "ncall_me: int =10000"
 
     def extract(s: str) -> str:
-        a = s.find(" ")
-        b = s.find("=")
-        if a < 0:
-            a = 0
-        if b < 0:
-            b = len(s)
-        return s[a:b].strip()
-
-    #   "iterable", "default", "key"
+        i = s.find("=")
+        if i >= 0:
+            s = s[:i]
+        i = s.find(":")
+        if i >= 0:
+            return s[:i].strip()
+        s = s.strip()
+        i = s.find(" ")
+        if i >= 0:
+            return s[i:].strip()
+        return s
+
+    #   "iterable", "*", "default", "key"
     #   "arg1", "arg2", "key"
     #   "ncall_me", "resume", "nsplit"
+    #   "ncall_me"
     return {extract(x): None for x in items if x != "*"}
 
 
+def _get_limit(annotation: Union[type, Annotated[float, Any], str]):
+    from iminuit import typing
+
+    if isinstance(annotation, str):
+        # This provides limited support for string annotations, which
+        # have a lot of problems, see https://peps.python.org/pep-0649.
+        try:
+            annotation = eval(annotation, None, typing.__dict__)
+        except NameError:
+            # We ignore unknown annotations to fix issue #846.
+            # I cannot replicate here what inspect.signature(..., eval_str=True) does.
+            # I need a dict with the global objects at the call site of describe, but
+            # it is not globals(). Anyway, when using strings, only the annotations
+            # from the package annotated-types are supported.
+            pass
+
+    if annotation == inspect.Parameter.empty:
+        return None
+
+    if get_origin(annotation) is not Annotated:
+        return None
+
+    tp, *constraints = get_args(annotation)
+    assert tp is float
+    lim = [-np.inf, np.inf]
+    for c in constraints:
+        if isinstance(c, slice):
+            if c.start is not None:
+                lim[0] = c.start
+            if c.stop is not None:
+                lim[1] = c.stop
+        # Minuit does not distinguish between closed and open intervals.
+        # We use a chain of ifs so that the code also works with the
+        # `Interval` class, which contains several of those attributes
+        # and which can be None.
+        gt = getattr(c, "gt", None)
+        ge = getattr(c, "ge", None)
+        lt = getattr(c, "lt", None)
+        le = getattr(c, "le", None)
+        if gt is not None:
+            lim[0] = gt
+        if ge is not None:
+            lim[0] = ge
+        if lt is not None:
+            lim[1] = lt
+        if le is not None:
+            lim[1] = le
+    return tuple(lim)
+
+
 def _guess_initial_step(val: float) -> float:
     return 1e-2 * abs(val) if val != 0 else 1e-1  # heuristic
 

tests/test_describe.py

@@ -165,7 +165,11 @@ def bar(self, *args):
             """Foo.bar(self, int ncall_me =10000, [resume=True, int nsplit=1])"""
             pass
 
+        def baz(self, *args):
+            """Foo.baz(self: Foo, ncall_me: int =10000, arg: np.ndarray = [])"""
+
     assert describe(Foo().bar) == ["ncall_me", "resume", "nsplit"]
+    assert describe(Foo().baz) == ["ncall_me", "arg"]
 
 
 def test_from_docstring_3():