Merge #2784 #2827

2784: Automatically generate `__text_signature__` for all functions r=davidhewitt a=davidhewitt

This PR makes it so that PyO3 generates `__text_signature__` by default for all functions. It also introduces `#[pyo3(text_signature = false)]` to disable the built-in generation.

There are a few limitations which we can improve later:
 - All default values are currently set to `...`. I think this is ok because `.pyi` files often do the same. Maybe for numbers, strings, `None` and `True`/`False` we could render these in a future PR.
 - No support for `#[new]` yet.

Alternative design ideas:
- Only autogenerate for methods with `#[pyo3(signature = (...))]` annotation. I started with this, and then decided it made sense to do it for everything.
- Opt-out with `#[pyo3(text_signature = None)]`. This is slightly harder to parse in the macro, but matches the final result in Python better, so if this looks preferable to others, I can change from `text_signature = false` to `text_signature = None`.

There's some small tidying up / refactoring to do before this merges (happy to take suggestions on this), however the general logic, design and docs are ready for review.


2827: pypy: enable `PyList::get_item_unchecked` r=davidhewitt a=davidhewitt

Split out from #2826. Approved previously as part of that review.

Co-authored-by: David Hewitt <1939362+davidhewitt@users.noreply.github.com>

guide/src/function.md

@@ -73,39 +73,7 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python
 
   - <a name="text_signature"></a> `#[pyo3(text_signature = "...")]`
 
-    Sets the function signature visible in Python tooling (such as via [`inspect.signature`]).
-
-    The example below creates a function `add` which has a signature describing two positional-only
-    arguments `a` and `b`.
-
-    ```rust
-    use pyo3::prelude::*;
-
-    /// This function adds two unsigned 64-bit integers.
-    #[pyfunction]
-    #[pyo3(text_signature = "(a, b, /)")]
-    fn add(a: u64, b: u64) -> u64 {
-        a + b
-    }
-    #
-    # fn main() -> PyResult<()> {
-    #     Python::with_gil(|py| {
-    #         let fun = pyo3::wrap_pyfunction!(add, py)?;
-    #
-    #         let doc: String = fun.getattr("__doc__")?.extract()?;
-    #         assert_eq!(doc, "This function adds two unsigned 64-bit integers.");
-    #
-    #         let inspect = PyModule::import(py, "inspect")?.getattr("signature")?;
-    #         let sig: String = inspect
-    #             .call1((fun,))?
-    #             .call_method0("__str__")?
-    #             .extract()?;
-    #         assert_eq!(sig, "(a, b, /)");
-    #
-    #         Ok(())
-    #     })
-    # }
-    ```
+    Overrides the PyO3-generated function signature visible in Python tooling (such as via [`inspect.signature`]). See the [corresponding topic in the Function Signatures subchapter](./function/signature.md#making-the-function-signature-available-to-python).
 
   - <a name="pass_module" ></a> `#[pyo3(pass_module)]`
 
@@ -161,47 +129,6 @@ The `#[pyo3]` attribute can be used on individual arguments to modify properties
 
 ## Advanced function patterns
 
-### Making the function signature available to Python (old method)
-
-Alternatively, simply make sure the first line of your docstring is
-formatted like in the following example. Please note that the newline after the
-`--` is mandatory. The `/` signifies the end of positional-only arguments.
-
-`#[pyo3(text_signature)]` should be preferred, since it will override automatically
-generated signatures when those are added in a future version of PyO3.
-
-```rust
-# #![allow(dead_code)]
-use pyo3::prelude::*;
-
-/// add(a, b, /)
-/// --
-///
-/// This function adds two unsigned 64-bit integers.
-#[pyfunction]
-fn add(a: u64, b: u64) -> u64 {
-    a + b
-}
-
-// a function with a signature but without docs. Both blank lines after the `--` are mandatory.
-
-/// sub(a, b, /)
-/// --
-#[pyfunction]
-fn sub(a: u64, b: u64) -> u64 {
-    a - b
-}
-```
-
-When annotated like this, signatures are also correctly displayed in IPython.
-
-```text
->>> pyo3_test.add?
-Signature: pyo3_test.add(a, b, /)
-Docstring: This function adds two unsigned 64-bit integers.
-Type:      builtin_function_or_method
-```
-
 ### Calling Python functions in Rust
 
 You can pass Python `def`'d functions and built-in functions to Rust functions [`PyFunction`]

guide/src/function/signature.md

@@ -188,3 +188,138 @@ impl MyClass {
     }
 }
 ```
+
+## Making the function signature available to Python
+
+The function signature is exposed to Python via the `__text_signature__` attribute. PyO3 automatically generates this for every `#[pyfunction]` and all `#[pymethods]` directly from the Rust function, taking into account any override done with the `#[pyo3(signature = (...))]` option.
+
+This automatic generation has some limitations, which may be improved in the future:
+- It will not include the value of default arguments, replacing them all with `...`. (`.pyi` type stub files commonly also use `...` for all default arguments in the same way.)
+- Nothing is generated for the `#[new]` method of a `#[pyclass]`.
+
+In cases where the automatically-generated signature needs adjusting, it can [be overridden](#overriding-the-generated-signature) using the `#[pyo3(text_signature)]` option.)
+
+The example below creates a function `add` which accepts two positional-only arguments `a` and `b`, where `b` has a default value of zero.
+
+```rust
+use pyo3::prelude::*;
+
+/// This function adds two unsigned 64-bit integers.
+#[pyfunction]
+#[pyo3(signature = (a, b=0, /))]
+fn add(a: u64, b: u64) -> u64 {
+    a + b
+}
+#
+# fn main() -> PyResult<()> {
+#     Python::with_gil(|py| {
+#         let fun = pyo3::wrap_pyfunction!(add, py)?;
+#
+#         let doc: String = fun.getattr("__doc__")?.extract()?;
+#         assert_eq!(doc, "This function adds two unsigned 64-bit integers.");
+#
+#         let inspect = PyModule::import(py, "inspect")?.getattr("signature")?;
+#         let sig: String = inspect
+#             .call1((fun,))?
+#             .call_method0("__str__")?
+#             .extract()?;
+#
+#         #[cfg(Py_3_8)]  // on 3.7 the signature doesn't render b, upstream bug?
+#         assert_eq!(sig, "(a, b=Ellipsis, /)");
+#
+#         Ok(())
+#     })
+# }
+```
+
+The following IPython output demonstrates how this generated signature will be seen from Python tooling:
+
+```text
+>>> pyo3_test.add.__text_signature__
+'(a, b=..., /)'
+>>> pyo3_test.add?
+Signature: pyo3_test.add(a, b=Ellipsis, /)
+Docstring: This function adds two unsigned 64-bit integers.
+Type:      builtin_function_or_method
+```
+
+### Overriding the generated signature
+
+The `#[pyo3(text_signature = "(<some signature>)")]` attribute can be used to override the default generated signature.
+
+In the snippet below, the text signature attribute is used to include the default value of `0` for the argument `b`, instead of the automatically-generated default value of `...`:
+
+```rust
+use pyo3::prelude::*;
+
+/// This function adds two unsigned 64-bit integers.
+#[pyfunction]
+#[pyo3(signature = (a, b=0, /), text_signature = "(a, b=0, /)")]
+fn add(a: u64, b: u64) -> u64 {
+    a + b
+}
+#
+# fn main() -> PyResult<()> {
+#     Python::with_gil(|py| {
+#         let fun = pyo3::wrap_pyfunction!(add, py)?;
+#
+#         let doc: String = fun.getattr("__doc__")?.extract()?;
+#         assert_eq!(doc, "This function adds two unsigned 64-bit integers.");
+#
+#         let inspect = PyModule::import(py, "inspect")?.getattr("signature")?;
+#         let sig: String = inspect
+#             .call1((fun,))?
+#             .call_method0("__str__")?
+#             .extract()?;
+#         assert_eq!(sig, "(a, b=0, /)");
+#
+#         Ok(())
+#     })
+# }
+```
+
+PyO3 will include the contents of the annotation unmodified as the `__text_signature`. Below shows how IPython will now present this (see the default value of 0 for b):
+
+```text
+>>> pyo3_test.add.__text_signature__
+'(a, b=0, /)'
+>>> pyo3_test.add?
+Signature: pyo3_test.add(a, b=0, /)
+Docstring: This function adds two unsigned 64-bit integers.
+Type:      builtin_function_or_method
+```
+
+If no signature is wanted at all, `#[pyo3(text_signature = None)]` will disable the built-in signature. The snippet below demonstrates use of this:
+
+```rust
+use pyo3::prelude::*;
+
+/// This function adds two unsigned 64-bit integers.
+#[pyfunction]
+#[pyo3(signature = (a, b=0, /), text_signature = None)]
+fn add(a: u64, b: u64) -> u64 {
+    a + b
+}
+#
+# fn main() -> PyResult<()> {
+#     Python::with_gil(|py| {
+#         let fun = pyo3::wrap_pyfunction!(add, py)?;
+#
+#         let doc: String = fun.getattr("__doc__")?.extract()?;
+#         assert_eq!(doc, "This function adds two unsigned 64-bit integers.");
+#         assert!(fun.getattr("__text_signature__")?.is_none());
+#
+#         Ok(())
+#     })
+# }
+```
+
+Now the function's `__text_signature__` will be set to `None`, and IPython will not display any signature in the help:
+
+```text
+>>> pyo3_test.add.__text_signature__ == None
+True
+>>> pyo3_test.add?
+Docstring: This function adds two unsigned 64-bit integers.
+Type:      builtin_function_or_method
+```

newsfragments/2784.changed.md

@@ -0,0 +1 @@
+Automatically generate `__text_signature__` for all Python functions created using `#[pyfunction]` and `#[pymethods]`.

newsfragments/2827.added.md

@@ -0,0 +1 @@
+Add `PyList::get_item_unchecked` for PyPy.

pyo3-macros-backend/src/attributes.rs

@@ -81,11 +81,46 @@ impl ToTokens for NameLitStr {
     }
 }
 
+/// Text signatue can be either a literal string or opt-in/out
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum TextSignatureAttributeValue {
+    Str(LitStr),
+    // `None` ident to disable automatic text signature generation
+    Disabled(Ident),
+}
+
+impl Parse for TextSignatureAttributeValue {
+    fn parse(input: ParseStream<'_>) -> Result<Self> {
+        if let Ok(lit_str) = input.parse::<LitStr>() {
+            return Ok(TextSignatureAttributeValue::Str(lit_str));
+        }
+
+        let err_span = match input.parse::<Ident>() {
+            Ok(ident) if ident == "None" => {
+                return Ok(TextSignatureAttributeValue::Disabled(ident));
+            }
+            Ok(other_ident) => other_ident.span(),
+            Err(e) => e.span(),
+        };
+
+        Err(err_spanned!(err_span => "expected a string literal or `None`"))
+    }
+}
+
+impl ToTokens for TextSignatureAttributeValue {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        match self {
+            TextSignatureAttributeValue::Str(s) => s.to_tokens(tokens),
+            TextSignatureAttributeValue::Disabled(b) => b.to_tokens(tokens),
+        }
+    }
+}
+
 pub type ExtendsAttribute = KeywordAttribute<kw::extends, Path>;
 pub type FreelistAttribute = KeywordAttribute<kw::freelist, Box<Expr>>;
 pub type ModuleAttribute = KeywordAttribute<kw::module, LitStr>;
 pub type NameAttribute = KeywordAttribute<kw::name, NameLitStr>;
-pub type TextSignatureAttribute = KeywordAttribute<kw::text_signature, LitStr>;
+pub type TextSignatureAttribute = KeywordAttribute<kw::text_signature, TextSignatureAttributeValue>;
 
 impl<K: Parse + std::fmt::Debug, V: Parse> Parse for KeywordAttribute<K, V> {
     fn parse(input: ParseStream<'_>) -> Result<Self> {

pyo3-macros-backend/src/method.rs

@@ -5,7 +5,9 @@ use std::borrow::Cow;
 use crate::attributes::TextSignatureAttribute;
 use crate::deprecations::{Deprecation, Deprecations};
 use crate::params::impl_arg_params;
-use crate::pyfunction::{DeprecatedArgs, FunctionSignature, PyFunctionArgPyO3Attributes};
+use crate::pyfunction::{
+    text_signature_or_auto, DeprecatedArgs, FunctionSignature, PyFunctionArgPyO3Attributes,
+};
 use crate::pyfunction::{PyFunctionOptions, SignatureAttribute};
 use crate::utils::{self, PythonDoc};
 use proc_macro2::{Span, TokenStream};
@@ -202,7 +204,7 @@ impl CallingConvention {
     pub fn from_signature(signature: &FunctionSignature<'_>) -> Self {
         if signature.python_signature.has_no_args() {
             Self::Noargs
-        } else if signature.python_signature.accepts_kwargs {
+        } else if signature.python_signature.kwargs.is_some() {
             // for functions that accept **kwargs, always prefer varargs
             Self::Varargs
         } else if cfg!(not(feature = "abi3")) {
@@ -288,13 +290,6 @@ impl<'a> FnSpec<'a> {
         let ty = get_return_info(&sig.output);
         let python_name = python_name.as_ref().unwrap_or(name).unraw();
 
-        let doc = utils::get_doc(
-            meth_attrs,
-            text_signature
-                .as_ref()
-                .map(|attr| (Cow::Borrowed(&python_name), attr)),
-        );
-
         let arguments: Vec<_> = if skip_first_arg {
             sig.inputs
                 .iter_mut()
@@ -320,6 +315,16 @@ impl<'a> FnSpec<'a> {
             FunctionSignature::from_arguments(arguments)
         };
 
+        let text_signature_string = match &fn_type {
+            FnType::FnNew | FnType::Getter(_) | FnType::Setter(_) | FnType::ClassAttribute => None,
+            _ => text_signature_or_auto(text_signature.as_ref(), &signature, &fn_type),
+        };
+
+        let doc = utils::get_doc(
+            meth_attrs,
+            text_signature_string.map(|sig| (Cow::Borrowed(&python_name), sig)),
+        );
+
         let convention =
             fixed_convention.unwrap_or_else(|| CallingConvention::from_signature(&signature));
 

pyo3-macros-backend/src/params.rs

@@ -82,12 +82,12 @@ pub fn impl_arg_params(
         .map(|arg| impl_arg_param(arg, &mut option_pos, py, &args_array))
         .collect::<Result<_>>()?;
 
-    let args_handler = if spec.signature.python_signature.accepts_varargs {
+    let args_handler = if spec.signature.python_signature.varargs.is_some() {
         quote! { _pyo3::impl_::extract_argument::TupleVarargs }
     } else {
         quote! { _pyo3::impl_::extract_argument::NoVarargs }
     };
-    let kwargs_handler = if spec.signature.python_signature.accepts_kwargs {
+    let kwargs_handler = if spec.signature.python_signature.kwargs.is_some() {
         quote! { _pyo3::impl_::extract_argument::DictVarkeywords }
     } else {
         quote! { _pyo3::impl_::extract_argument::NoVarkeywords }

pyo3-macros-backend/src/pyclass.rs

@@ -9,6 +9,7 @@ use crate::attributes::{
 use crate::deprecations::{Deprecation, Deprecations};
 use crate::konst::{ConstAttributes, ConstSpec};
 use crate::method::FnSpec;
+use crate::pyfunction::text_signature_or_none;
 use crate::pyimpl::{gen_py_const, PyClassMethodsType};
 use crate::pymethod::{
     impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, MethodAndSlotDef, PropertyType,
@@ -198,12 +199,10 @@ pub fn build_py_class(
     methods_type: PyClassMethodsType,
 ) -> syn::Result<TokenStream> {
     args.options.take_pyo3_options(&mut class.attrs)?;
+    let text_signature_string = text_signature_or_none(args.options.text_signature.as_ref());
     let doc = utils::get_doc(
         &class.attrs,
-        args.options
-            .text_signature
-            .as_ref()
-            .map(|attr| (get_class_python_name(&class.ident, &args), attr)),
+        text_signature_string.map(|s| (get_class_python_name(&class.ident, &args), s)),
     );
     let krate = get_pyo3_crate(&args.options.krate);
 
@@ -461,12 +460,11 @@ pub fn build_py_enum(
         bail_spanned!(enum_.brace_token.span => "#[pyclass] can't be used on enums without any variants");
     }
 
+    let text_signature_string = text_signature_or_none(args.options.text_signature.as_ref());
+
     let doc = utils::get_doc(
         &enum_.attrs,
-        args.options
-            .text_signature
-            .as_ref()
-            .map(|attr| (get_class_python_name(&enum_.ident, &args), attr)),
+        text_signature_string.map(|s| (get_class_python_name(&enum_.ident, &args), s)),
     );
     let enum_ = PyClassEnum::new(enum_)?;
     impl_enum(enum_, &args, doc, method_type)