Merge branch 'main' into frompyobject

Cargo.toml

@@ -65,6 +65,9 @@ pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.0-dev", featu
 [features]
 default = ["macros"]
 
+# Enables support for `async fn` for `#[pyfunction]` and `#[pymethods]`.
+experimental-async = ["macros", "pyo3-macros/experimental-async"]
+
 # Enables pyo3::inspect module and additional type information on FromPyObject
 # and IntoPy traits
 experimental-inspect = []
@@ -115,8 +118,9 @@ full = [
     "chrono",
     "chrono-tz",
     "either",
-    "experimental-inspect",
+    "experimental-async",
     "experimental-declarative-modules",
+    "experimental-inspect",
     "eyre",
     "hashbrown",
     "indexmap",

guide/src/async-await.md

@@ -6,6 +6,7 @@
 
 ```rust
 # #![allow(dead_code)]
+# #[cfg(feature = "experimental-async")] {
 use std::{thread, time::Duration};
 use futures::channel::oneshot;
 use pyo3::prelude::*;
@@ -20,6 +21,7 @@ async fn sleep(seconds: f64, result: Option<PyObject>) -> Option<PyObject> {
     rx.await.unwrap();
     result
 }
+# }
 ```
 
 *Python awaitables instantiated with this method can only be awaited in *asyncio* context. Other Python async runtime may be supported in the future.*
@@ -72,6 +74,7 @@ Cancellation on the Python side can be caught using [`CancelHandle`]({{#PYO3_DOC
 
 ```rust
 # #![allow(dead_code)]
+# #[cfg(feature = "experimental-async")] {
 use futures::FutureExt;
 use pyo3::prelude::*;
 use pyo3::coroutine::CancelHandle;
@@ -83,11 +86,12 @@ async fn cancellable(#[pyo3(cancel_handle)] mut cancel: CancelHandle) {
         _ = cancel.cancelled().fuse() => println!("cancelled"),
     }
 }
+# }
 ```
 
 ## The `Coroutine` type
 
-To make a Rust future awaitable in Python, PyO3 defines a [`Coroutine`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.Coroutine.html) type, which implements the Python [coroutine protocol](https://docs.python.org/3/library/collections.abc.html#collections.abc.Coroutine). 
+To make a Rust future awaitable in Python, PyO3 defines a [`Coroutine`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.Coroutine.html) type, which implements the Python [coroutine protocol](https://docs.python.org/3/library/collections.abc.html#collections.abc.Coroutine).
 
 Each `coroutine.send` call is translated to a `Future::poll` call. If a [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) parameter is declared, the exception passed to `coroutine.throw` call is stored in it and can be retrieved with [`CancelHandle::cancelled`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html#method.cancelled); otherwise, it cancels the Rust future, and the exception is reraised;
 

guide/src/features.md

@@ -51,6 +51,12 @@ If you do not enable this feature, you should call `pyo3::prepare_freethreaded_p
 
 ## Advanced Features
 
+### `experimental-async`
+
+This feature adds support for `async fn` in `#[pyfunction]` and `#[pymethods]`.
+
+The feature has some unfinished refinements and performance improvements. To help finish this off, see [issue #1632](https://github.com/PyO3/pyo3/issues/1632) and its associated draft PRs.
+
 ### `experimental-inspect`
 
 This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types.

newsfragments/3931.added.md

@@ -0,0 +1 @@
+Add `experimental-async` feature.

pyo3-macros-backend/Cargo.toml

@@ -26,3 +26,6 @@ features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-trait
 
 [lints]
 workspace = true
+
+[features]
+experimental-async = []

pyo3-macros-backend/src/method.rs

@@ -512,6 +512,13 @@ impl<'a> FnSpec<'a> {
             }
         }
 
+        if self.asyncness.is_some() {
+            ensure_spanned!(
+                cfg!(feature = "experimental-async"),
+                self.asyncness.span() => "async functions are only supported with the `experimental-async` feature"
+            );
+        }
+
         let rust_call = |args: Vec<TokenStream>, holders: &mut Vec<TokenStream>| {
             let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise, holders, ctx);
 

pyo3-macros-backend/src/module.rs

@@ -115,19 +115,10 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result<TokenStream> {
     for item in &mut *items {
         match item {
             Item::Use(item_use) => {
-                let mut is_pyo3 = false;
-                item_use.attrs.retain(|attr| {
-                    let found = attr.path().is_ident("pymodule_export");
-                    is_pyo3 |= found;
-                    !found
-                });
-                if is_pyo3 {
-                    let cfg_attrs = item_use
-                        .attrs
-                        .iter()
-                        .filter(|attr| attr.path().is_ident("cfg"))
-                        .cloned()
-                        .collect::<Vec<_>>();
+                let is_pymodule_export =
+                    find_and_remove_attribute(&mut item_use.attrs, "pymodule_export");
+                if is_pymodule_export {
+                    let cfg_attrs = get_cfg_attributes(&item_use.attrs);
                     extract_use_items(
                         &item_use.tree,
                         &cfg_attrs,
@@ -137,23 +128,116 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result<TokenStream> {
                 }
             }
             Item::Fn(item_fn) => {
-                let mut is_module_init = false;
-                item_fn.attrs.retain(|attr| {
-                    let found = attr.path().is_ident("pymodule_init");
-                    is_module_init |= found;
-                    !found
-                });
-                if is_module_init {
-                    ensure_spanned!(pymodule_init.is_none(), item_fn.span() => "only one pymodule_init may be specified");
-                    let ident = &item_fn.sig.ident;
+                ensure_spanned!(
+                    !has_attribute(&item_fn.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+                let is_pymodule_init =
+                    find_and_remove_attribute(&mut item_fn.attrs, "pymodule_init");
+                let ident = &item_fn.sig.ident;
+                if is_pymodule_init {
+                    ensure_spanned!(
+                        !has_attribute(&item_fn.attrs, "pyfunction"),
+                        item_fn.span() => "`#[pyfunction]` cannot be used alongside `#[pymodule_init]`"
+                    );
+                    ensure_spanned!(pymodule_init.is_none(), item_fn.span() => "only one `#[pymodule_init]` may be specified");
                     pymodule_init = Some(quote! { #ident(module)?; });
-                } else {
-                    bail_spanned!(item.span() => "only 'use' statements and and pymodule_init functions are allowed in #[pymodule]")
+                } else if has_attribute(&item_fn.attrs, "pyfunction") {
+                    module_items.push(ident.clone());
+                    module_items_cfg_attrs.push(get_cfg_attributes(&item_fn.attrs));
                 }
             }
-            item => {
-                bail_spanned!(item.span() => "only 'use' statements and and pymodule_init functions are allowed in #[pymodule]")
+            Item::Struct(item_struct) => {
+                ensure_spanned!(
+                    !has_attribute(&item_struct.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+                if has_attribute(&item_struct.attrs, "pyclass") {
+                    module_items.push(item_struct.ident.clone());
+                    module_items_cfg_attrs.push(get_cfg_attributes(&item_struct.attrs));
+                }
+            }
+            Item::Enum(item_enum) => {
+                ensure_spanned!(
+                    !has_attribute(&item_enum.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+                if has_attribute(&item_enum.attrs, "pyclass") {
+                    module_items.push(item_enum.ident.clone());
+                    module_items_cfg_attrs.push(get_cfg_attributes(&item_enum.attrs));
+                }
+            }
+            Item::Mod(item_mod) => {
+                ensure_spanned!(
+                    !has_attribute(&item_mod.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+                if has_attribute(&item_mod.attrs, "pymodule") {
+                    module_items.push(item_mod.ident.clone());
+                    module_items_cfg_attrs.push(get_cfg_attributes(&item_mod.attrs));
+                }
+            }
+            Item::ForeignMod(item) => {
+                ensure_spanned!(
+                    !has_attribute(&item.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+            }
+            Item::Trait(item) => {
+                ensure_spanned!(
+                    !has_attribute(&item.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+            }
+            Item::Const(item) => {
+                ensure_spanned!(
+                    !has_attribute(&item.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+            }
+            Item::Static(item) => {
+                ensure_spanned!(
+                    !has_attribute(&item.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+            }
+            Item::Macro(item) => {
+                ensure_spanned!(
+                    !has_attribute(&item.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+            }
+            Item::ExternCrate(item) => {
+                ensure_spanned!(
+                    !has_attribute(&item.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+            }
+            Item::Impl(item) => {
+                ensure_spanned!(
+                    !has_attribute(&item.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
             }
+            Item::TraitAlias(item) => {
+                ensure_spanned!(
+                    !has_attribute(&item.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+            }
+            Item::Type(item) => {
+                ensure_spanned!(
+                    !has_attribute(&item.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+            }
+            Item::Union(item) => {
+                ensure_spanned!(
+                    !has_attribute(&item.attrs, "pymodule_export"),
+                    item.span() => "`#[pymodule_export]` may only be used on `use` statements"
+                );
+            }
+            _ => (),
         }
     }
 
@@ -355,6 +439,31 @@ fn get_pyfn_attr(attrs: &mut Vec<syn::Attribute>) -> syn::Result<Option<PyFnArgs
     Ok(pyfn_args)
 }
 
+fn get_cfg_attributes(attrs: &[syn::Attribute]) -> Vec<syn::Attribute> {
+    attrs
+        .iter()
+        .filter(|attr| attr.path().is_ident("cfg"))
+        .cloned()
+        .collect()
+}
+
+fn find_and_remove_attribute(attrs: &mut Vec<syn::Attribute>, ident: &str) -> bool {
+    let mut found = false;
+    attrs.retain(|attr| {
+        if attr.path().is_ident(ident) {
+            found = true;
+            false
+        } else {
+            true
+        }
+    });
+    found
+}
+
+fn has_attribute(attrs: &[syn::Attribute], ident: &str) -> bool {
+    attrs.iter().any(|attr| attr.path().is_ident(ident))
+}
+
 enum PyModulePyO3Option {
     Crate(CrateAttribute),
     Name(NameAttribute),

pyo3-macros/Cargo.toml

@@ -15,6 +15,7 @@ proc-macro = true
 
 [features]
 multiple-pymethods = []
+experimental-async = ["pyo3-macros-backend/experimental-async"]
 experimental-declarative-modules = []
 
 [dependencies]

src/impl_.rs

@@ -6,7 +6,7 @@
 //! APIs may may change at any time without documentation in the CHANGELOG and without
 //! breaking semver guarantees.
 
-#[cfg(feature = "macros")]
+#[cfg(feature = "experimental-async")]
 pub mod coroutine;
 pub mod deprecations;
 pub mod extract_argument;

src/lib.rs

@@ -418,7 +418,7 @@ pub mod buffer;
 pub mod callback;
 pub mod conversion;
 mod conversions;
-#[cfg(feature = "macros")]
+#[cfg(feature = "experimental-async")]
 pub mod coroutine;
 #[macro_use]
 #[doc(hidden)]

tests/test_compile_error.rs

@@ -28,7 +28,8 @@ fn test_compile_errors() {
     t.compile_fail("tests/ui/wrong_aspyref_lifetimes.rs");
     t.compile_fail("tests/ui/invalid_pyfunctions.rs");
     t.compile_fail("tests/ui/invalid_pymethods.rs");
-    #[cfg(Py_LIMITED_API)]
+    // output changes with async feature
+    #[cfg(all(Py_LIMITED_API, feature = "experimental-async"))]
     t.compile_fail("tests/ui/abi3_nativetype_inheritance.rs");
     t.compile_fail("tests/ui/invalid_intern_arg.rs");
     t.compile_fail("tests/ui/invalid_frozen_pyclass_borrow.rs");
@@ -49,4 +50,6 @@ fn test_compile_errors() {
     t.compile_fail("tests/ui/invalid_pymodule_trait.rs");
     #[cfg(feature = "experimental-declarative-modules")]
     t.compile_fail("tests/ui/invalid_pymodule_two_pymodule_init.rs");
+    #[cfg(feature = "experimental-async")]
+    t.compile_fail("tests/ui/invalid_cancel_handle.rs");
 }

tests/test_coroutine.rs

@@ -1,4 +1,4 @@
-#![cfg(feature = "macros")]
+#![cfg(feature = "experimental-async")]
 #![cfg(not(target_arch = "wasm32"))]
 use std::{task::Poll, thread, time::Duration};
 

tests/test_declarative_module.rs

@@ -15,8 +15,8 @@ struct ValueClass {
 #[pymethods]
 impl ValueClass {
     #[new]
-    fn new(value: usize) -> ValueClass {
-        ValueClass { value }
+    fn new(value: usize) -> Self {
+        Self { value }
     }
 }
 
@@ -48,6 +48,33 @@ mod declarative_module {
     #[pymodule_export]
     use super::{declarative_module2, double, MyError, ValueClass as Value};
 
+    #[pymodule]
+    mod inner {
+        use super::*;
+
+        #[pyfunction]
+        fn triple(x: usize) -> usize {
+            x * 3
+        }
+
+        #[pyclass]
+        struct Struct;
+
+        #[pymethods]
+        impl Struct {
+            #[new]
+            fn new() -> Self {
+                Self
+            }
+        }
+
+        #[pyclass]
+        enum Enum {
+            A,
+            B,
+        }
+    }
+
     #[pymodule_init]
     fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
         m.add("double2", m.getattr("double")?)
@@ -65,7 +92,6 @@ mod declarative_submodule {
     use super::{double, double_value};
 }
 
-/// A module written using declarative syntax.
 #[pymodule]
 #[pyo3(name = "declarative_module_renamed")]
 mod declarative_module2 {
@@ -84,7 +110,7 @@ fn test_declarative_module() {
         );
 
         py_assert!(py, m, "m.double(2) == 4");
-        py_assert!(py, m, "m.double2(3) == 6");
+        py_assert!(py, m, "m.inner.triple(3) == 9");
         py_assert!(py, m, "m.declarative_submodule.double(4) == 8");
         py_assert!(
             py,
@@ -97,5 +123,7 @@ fn test_declarative_module() {
         py_assert!(py, m, "not hasattr(m, 'LocatedClass')");
         #[cfg(not(Py_LIMITED_API))]
         py_assert!(py, m, "hasattr(m, 'LocatedClass')");
+        py_assert!(py, m, "isinstance(m.inner.Struct(), m.inner.Struct)");
+        py_assert!(py, m, "isinstance(m.inner.Enum.A, m.inner.Enum)");
     })
 }