From 720893a62151d00223478072d504f6102cf2866b Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 2 Jun 2023 08:09:22 +0100 Subject: [PATCH 01/59] update PR template to detail state of licensing --- .github/pull_request_template.md | 6 +++++- README.md | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4473a007e8f..f9877a1ca40 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,8 @@ -Thank you for contributing to pyo3! +Thank you for contributing to PyO3! + +By submitting these contributions you agree for them to be licensed under PyO3's [Apache-2.0 license](https://github.com/PyO3/pyo3#license). + +PyO3 is currently undergoing a relicensing process to match Rust's dual-license under `Apache-2.0` and `MIT` licenses. While that process is ongoing, if you are a first-time contributor please add your agreement as a comment in [#3108](https://github.com/PyO3/pyo3/pull/3108). Please consider adding the following to your pull request: - an entry for this PR in newsfragments - see [https://pyo3.rs/main/contributing.html#documenting-changes] diff --git a/README.md b/README.md index 594fbc7f519..cbfe3306e25 100644 --- a/README.md +++ b/README.md @@ -244,7 +244,8 @@ If you don't have time to contribute yourself but still wish to support the proj ## License -PyO3 is licensed under the [Apache-2.0 license](https://opensource.org/licenses/APACHE-2.0). +PyO3 is licensed under the [Apache-2.0 license](https://opensource.org/licenses/APACHE-2.0). (The PyO3 project is in the process of collecting permission from past contributors to additionally license under the [MIT license](https://opensource.org/license/mit/), see [#3108](https://github.com/PyO3/pyo3/pull/3108). Once complete, PyO3 will additionally be licensed under the MIT license, the same as the Rust language itself is both Apache and MIT licensed.) + Python is licensed under the [Python License](https://docs.python.org/3/license.html). Deploys by Netlify From 4f5ee5f03d4251289740e7b0f6cf3130c34fcd4d Mon Sep 17 00:00:00 2001 From: Jake Lishman Date: Wed, 31 May 2023 16:51:40 +0100 Subject: [PATCH 02/59] Add `PyAny::lookup_special` `PyAny::lookup_special` is an approximate equivalent to the CPython internal `_PyObject_LookupSpecial`, which is used to resolve lookups of "magic" methods. These are only looked up from the type, and skip the instance dictionary during the lookup. Despite this, they are still required to resolve the descriptor protocol. Many magic methods have slots on the `PyTypeObject` or respective subobjects, but these are not necessarily available when targeting the limited API or PyPy. In these cases, the requisite logic can be worked around using safe but likely slower APIs. Co-authored-by: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Fix up lookup-special --- src/types/any.rs | 120 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/src/types/any.rs b/src/types/any.rs index afdeb6ab573..a545d9b2ac1 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -124,6 +124,51 @@ impl PyAny { } } + /// Retrieve an attribute value, skipping the instance dictionary during the lookup but still + /// binding the object to the instance. + /// + /// This is useful when trying to resolve Python's "magic" methods like `__getitem__`, which + /// are looked up starting from the type object. This returns an `Option` as it is not + /// typically a direct error for the special lookup to fail, as magic methods are optional in + /// many situations in which they might be called. + /// + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// to intern `attr_name`. + #[allow(dead_code)] // Currently only used with num-complex+abi3, so dead without that. + pub(crate) fn lookup_special(&self, attr_name: N) -> PyResult> + where + N: IntoPy>, + { + let py = self.py(); + let self_type = self.get_type(); + let attr = if let Ok(attr) = self_type.getattr(attr_name) { + attr + } else { + return Ok(None); + }; + + // Manually resolve descriptor protocol. + if cfg!(Py_3_10) + || unsafe { ffi::PyType_HasFeature(attr.get_type_ptr(), ffi::Py_TPFLAGS_HEAPTYPE) } != 0 + { + // This is the preferred faster path, but does not work on static types (generally, + // types defined in extension modules) before Python 3.10. + unsafe { + let descr_get_ptr = ffi::PyType_GetSlot(attr.get_type_ptr(), ffi::Py_tp_descr_get); + if descr_get_ptr.is_null() { + return Ok(Some(attr)); + } + let descr_get: ffi::descrgetfunc = std::mem::transmute(descr_get_ptr); + let ret = descr_get(attr.as_ptr(), self.as_ptr(), self_type.as_ptr()); + py.from_owned_ptr_or_err(ret).map(Some) + } + } else if let Ok(descr_get) = attr.get_type().getattr(crate::intern!(py, "__get__")) { + descr_get.call1((attr, self, self_type)).map(Some) + } else { + Ok(Some(attr)) + } + } + /// Sets an attribute value. /// /// This is equivalent to the Python expression `self.attr_name = value`. @@ -974,9 +1019,82 @@ impl PyAny { #[cfg(test)] mod tests { use crate::{ - types::{IntoPyDict, PyBool, PyList, PyLong, PyModule}, + types::{IntoPyDict, PyAny, PyBool, PyList, PyLong, PyModule}, Python, ToPyObject, }; + + #[test] + fn test_lookup_special() { + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class CustomCallable: + def __call__(self): + return 1 + +class SimpleInt: + def __int__(self): + return 1 + +class InheritedInt(SimpleInt): pass + +class NoInt: pass + +class NoDescriptorInt: + __int__ = CustomCallable() + +class InstanceOverrideInt: + def __int__(self): + return 1 +instance_override = InstanceOverrideInt() +instance_override.__int__ = lambda self: 2 + +class ErrorInDescriptorInt: + @property + def __int__(self): + raise ValueError("uh-oh!") + +class NonHeapNonDescriptorInt: + # A static-typed callable that doesn't implement `__get__`. These are pretty hard to come by. + __int__ = int + "#, + "test.py", + "test", + ) + .unwrap(); + + let int = crate::intern!(py, "__int__"); + let eval_int = + |obj: &PyAny| obj.lookup_special(int)?.unwrap().call0()?.extract::(); + + let simple = module.getattr("SimpleInt").unwrap().call0().unwrap(); + assert_eq!(eval_int(simple).unwrap(), 1); + let inherited = module.getattr("InheritedInt").unwrap().call0().unwrap(); + assert_eq!(eval_int(inherited).unwrap(), 1); + let no_descriptor = module.getattr("NoDescriptorInt").unwrap().call0().unwrap(); + assert_eq!(eval_int(no_descriptor).unwrap(), 1); + let missing = module.getattr("NoInt").unwrap().call0().unwrap(); + assert!(missing.lookup_special(int).unwrap().is_none()); + // Note the instance override should _not_ call the instance method that returns 2, + // because that's not how special lookups are meant to work. + let instance_override = module.getattr("instance_override").unwrap(); + assert_eq!(eval_int(instance_override).unwrap(), 1); + let descriptor_error = module + .getattr("ErrorInDescriptorInt") + .unwrap() + .call0() + .unwrap(); + assert!(descriptor_error.lookup_special(int).is_err()); + let nonheap_nondescriptor = module + .getattr("NonHeapNonDescriptorInt") + .unwrap() + .call0() + .unwrap(); + assert_eq!(eval_int(nonheap_nondescriptor).unwrap(), 0); + }) + } + #[test] fn test_call_for_non_existing_method() { Python::with_gil(|py| { From 8c115757c1d289667fc19df8b55ed88187d85b5c Mon Sep 17 00:00:00 2001 From: Jake Lishman Date: Fri, 26 May 2023 13:49:05 +0100 Subject: [PATCH 03/59] Fix `abi3` conversion of `__complex__` classes Python classes that were not `complex` but implemented the `__complex__` magic would have that method called via `PyComplex_AsCComplex` when running against the full API, but the limited-API version `PyComplex_RealAsDouble` does not attempt this conversion. If the input object is not already complex, we can call the magic before proceeding. --- newsfragments/3185.fixed.md | 1 + src/conversions/num_complex.rs | 140 +++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 newsfragments/3185.fixed.md diff --git a/newsfragments/3185.fixed.md b/newsfragments/3185.fixed.md new file mode 100644 index 00000000000..6b3a594c91a --- /dev/null +++ b/newsfragments/3185.fixed.md @@ -0,0 +1 @@ +Fix conversion of classes implementing `__complex__` to `Complex` when using `abi3` or PyPy. diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index 217d862a542..df6b54b45bc 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -152,6 +152,18 @@ macro_rules! complex_conversion { #[cfg(any(Py_LIMITED_API, PyPy))] unsafe { + let obj = if obj.is_instance_of::() { + obj + } else if let Some(method) = + obj.lookup_special(crate::intern!(obj.py(), "__complex__"))? + { + method.call0()? + } else { + // `obj` might still implement `__float__` or `__index__`, which will be + // handled by `PyComplex_{Real,Imag}AsDouble`, including propagating any + // errors if those methods don't exist / raise exceptions. + obj + }; let ptr = obj.as_ptr(); let real = ffi::PyComplex_RealAsDouble(ptr); if real == -1.0 { @@ -172,6 +184,7 @@ complex_conversion!(f64); #[cfg(test)] mod tests { use super::*; + use crate::types::PyModule; #[test] fn from_complex() { @@ -197,4 +210,131 @@ mod tests { assert!(obj.extract::>(py).is_err()); }); } + #[test] + fn from_python_magic() { + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class A: + def __complex__(self): return 3.0+1.2j +class B: + def __float__(self): return 3.0 +class C: + def __index__(self): return 3 + "#, + "test.py", + "test", + ) + .unwrap(); + let from_complex = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + from_complex.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + let from_float = module.getattr("B").unwrap().call0().unwrap(); + assert_eq!( + from_float.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + // Before Python 3.8, `__index__` wasn't tried by `float`/`complex`. + #[cfg(Py_3_8)] + { + let from_index = module.getattr("C").unwrap().call0().unwrap(); + assert_eq!( + from_index.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + } + }) + } + #[test] + fn from_python_inherited_magic() { + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class First: pass +class ComplexMixin: + def __complex__(self): return 3.0+1.2j +class FloatMixin: + def __float__(self): return 3.0 +class IndexMixin: + def __index__(self): return 3 +class A(First, ComplexMixin): pass +class B(First, FloatMixin): pass +class C(First, IndexMixin): pass + "#, + "test.py", + "test", + ) + .unwrap(); + let from_complex = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + from_complex.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + let from_float = module.getattr("B").unwrap().call0().unwrap(); + assert_eq!( + from_float.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + #[cfg(Py_3_8)] + { + let from_index = module.getattr("C").unwrap().call0().unwrap(); + assert_eq!( + from_index.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + } + }) + } + #[test] + fn from_python_noncallable_descriptor_magic() { + // Functions and lambdas implement the descriptor protocol in a way that makes + // `type(inst).attr(inst)` equivalent to `inst.attr()` for methods, but this isn't the only + // way the descriptor protocol might be implemented. + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class A: + @property + def __complex__(self): + return lambda: 3.0+1.2j + "#, + "test.py", + "test", + ) + .unwrap(); + let obj = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + obj.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + }) + } + #[test] + fn from_python_nondescriptor_magic() { + // Magic methods don't need to implement the descriptor protocol, if they're callable. + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class MyComplex: + def __call__(self): return 3.0+1.2j +class A: + __complex__ = MyComplex() + "#, + "test.py", + "test", + ) + .unwrap(); + let obj = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + obj.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + }) + } } From c035d3114a965e24d614070a6f8feb3b753bec73 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 2 Jun 2023 09:39:29 +0900 Subject: [PATCH 04/59] Add abi3 + num_bigint conversion --- newsfragments/3198.added.md | 1 + src/conversions/num_bigint.rs | 81 +++++++++++++++++++++++++++++------ 2 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3198.added.md diff --git a/newsfragments/3198.added.md b/newsfragments/3198.added.md new file mode 100644 index 00000000000..2c6514a42f8 --- /dev/null +++ b/newsfragments/3198.added.md @@ -0,0 +1 @@ +Add support for `num-bigint` feature in combination with `abi3`. \ No newline at end of file diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index a8aec454040..08cc0fb11d5 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -2,7 +2,7 @@ // // based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython -#![cfg(all(feature = "num-bigint", not(any(Py_LIMITED_API))))] +#![cfg(feature = "num-bigint")] //! Conversions to and from [num-bigint](https://docs.rs/num-bigint)’s [`BigInt`] and [`BigUint`] types. //! //! This is useful for converting Python integers when they may not fit in Rust's built-in integer types. @@ -57,15 +57,16 @@ //! ``` use crate::{ - err, ffi, types::*, AsPyPointer, FromPyObject, IntoPy, Py, PyAny, PyErr, PyObject, PyResult, - Python, ToPyObject, + ffi, types::*, AsPyPointer, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, + ToPyObject, }; use num_bigint::{BigInt, BigUint}; use std::os::raw::{c_int, c_uchar}; +#[cfg(not(Py_LIMITED_API))] unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyResult<()> { - err::error_on_minusone( + crate::err::error_on_minusone( ob.py(), ffi::_PyLong_AsByteArray( ob.as_ptr() as *mut ffi::PyLongObject, @@ -77,13 +78,33 @@ unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyRe ) } +#[cfg(Py_LIMITED_API)] +unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyResult<()> { + use crate::intern; + let py = ob.py(); + let kwargs = if is_signed != 0 { + let kwargs = PyDict::new(py); + kwargs.set_item(intern!(py, "signed"), true)?; + Some(kwargs) + } else { + None + }; + let bytes_obj = ob + .getattr(intern!(py, "to_bytes"))? + .call((buffer.len(), "little"), kwargs)?; + let bytes: &PyBytes = bytes_obj.downcast_unchecked(); + buffer.copy_from_slice(bytes.as_bytes()); + Ok(()) +} + macro_rules! bigint_conversion { ($rust_ty: ty, $is_signed: expr, $to_bytes: path, $from_bytes: path) => { #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] impl ToPyObject for $rust_ty { + #[cfg(not(Py_LIMITED_API))] fn to_object(&self, py: Python<'_>) -> PyObject { + let bytes = $to_bytes(self); unsafe { - let bytes = $to_bytes(self); let obj = ffi::_PyLong_FromByteArray( bytes.as_ptr() as *const c_uchar, bytes.len(), @@ -93,6 +114,23 @@ macro_rules! bigint_conversion { PyObject::from_owned_ptr(py, obj) } } + + #[cfg(Py_LIMITED_API)] + fn to_object(&self, py: Python<'_>) -> PyObject { + let bytes = $to_bytes(self); + let bytes_obj = PyBytes::new(py, &bytes); + let kwargs = if $is_signed > 0 { + let kwargs = PyDict::new(py); + kwargs.set_item(crate::intern!(py, "signed"), true).unwrap(); + Some(kwargs) + } else { + None + }; + py.get_type::() + .call_method("from_bytes", (bytes_obj, "little"), kwargs) + .expect("int.from_bytes() failed during to_object()") // FIXME: #1813 or similar + .into() + } } #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] @@ -109,14 +147,33 @@ macro_rules! bigint_conversion { unsafe { let num: Py = Py::from_owned_ptr_or_err(py, ffi::PyNumber_Index(ob.as_ptr()))?; - let n_bits = ffi::_PyLong_NumBits(num.as_ptr()); - let n_bytes = if n_bits == (-1isize as usize) { - return Err(PyErr::fetch(py)); - } else if n_bits == 0 { - 0 - } else { - (n_bits - 1 + $is_signed) / 8 + 1 + + let n_bytes = { + cfg_if::cfg_if! { + if #[cfg(not(Py_LIMITED_API))] { + // fast path + let n_bits = ffi::_PyLong_NumBits(num.as_ptr()); + if n_bits == (-1isize as usize) { + return Err(crate::PyErr::fetch(py)); + } else if n_bits == 0 { + 0 + } else { + (n_bits - 1 + $is_signed) / 8 + 1 + } + } else { + // slow path + let n_bits_obj = num.getattr(py, crate::intern!(py, "bit_length"))?.call0(py)?; + let n_bits_int: &PyLong = n_bits_obj.downcast_unchecked(py); + let n_bits = n_bits_int.extract::()?; + if n_bits == 0 { + 0 + } else { + (n_bits - 1 + $is_signed) / 8 + 1 + } + } + } }; + if n_bytes <= 128 { let mut buffer = [0; 128]; extract(num.as_ref(py), &mut buffer[..n_bytes], $is_signed)?; From 955d686cc5fc81193292167bb0f22c95532a9ba6 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 2 Jun 2023 08:15:13 +0100 Subject: [PATCH 05/59] remove copyright headers from source files --- pyo3-macros-backend/src/lib.rs | 1 - pyo3-macros-backend/src/method.rs | 2 -- pyo3-macros-backend/src/module.rs | 1 - pyo3-macros-backend/src/params.rs | 2 -- pyo3-macros-backend/src/pyclass.rs | 2 -- pyo3-macros-backend/src/pyfunction.rs | 2 -- pyo3-macros-backend/src/pyimpl.rs | 2 -- pyo3-macros-backend/src/pymethod.rs | 2 -- pyo3-macros-backend/src/utils.rs | 1 - pyo3-macros/src/lib.rs | 1 - src/callback.rs | 2 -- src/conversion.rs | 2 -- src/conversions/num_bigint.rs | 4 ---- src/derive_utils.rs | 4 ---- src/err/mod.rs | 2 -- src/exceptions.rs | 2 -- src/gil.rs | 2 -- src/impl_/freelist.rs | 2 -- src/instance.rs | 1 - src/marker.rs | 4 ---- src/prelude.rs | 2 -- src/type_object.rs | 1 - src/types/boolobject.rs | 1 - src/types/bytearray.rs | 1 - src/types/capsule.rs | 1 - src/types/code.rs | 2 -- src/types/dict.rs | 2 -- src/types/floatob.rs | 3 --- src/types/frame.rs | 2 -- src/types/frozenset.rs | 3 --- src/types/iterator.rs | 4 ---- src/types/list.rs | 4 ---- src/types/mapping.rs | 2 -- src/types/mod.rs | 2 -- src/types/module.rs | 4 ---- src/types/num.rs | 4 ---- src/types/sequence.rs | 1 - src/types/set.rs | 3 --- src/types/slice.rs | 2 -- src/types/string.rs | 2 -- src/types/traceback.rs | 2 -- src/types/tuple.rs | 2 -- src/types/typeobject.rs | 4 ---- 43 files changed, 95 deletions(-) diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index 1ba9c8abd70..61cdbb630c0 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! This crate contains the implementation of the proc macro attributes #![warn(elided_lifetimes_in_paths, unused_lifetimes)] diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 09b8b26d1a8..a867a301378 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::attributes::{TextSignatureAttribute, TextSignatureAttributeValue}; use crate::deprecations::{Deprecation, Deprecations}; use crate::params::impl_arg_params; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index cb2fda00c0f..e6db700718f 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! Code generation for the function that initializes a python module and adds classes and function. use crate::{ diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 32a9be5d73a..b1e8538122c 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::{ method::{FnArg, FnSpec}, pyfunction::FunctionSignature, diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 7e7ac6b615f..f723da0951f 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::borrow::Cow; use crate::attributes::{ diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 8f56b3e916a..53188c7c072 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::{ attributes::{ self, get_pyo3_options, take_attributes, take_pyo3_options, CrateAttribute, diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index d0a1b6157cf..0ab0695eca5 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::collections::HashSet; use crate::{ diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 9689d863a44..76ebc947028 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::borrow::Cow; use crate::attributes::NameAttribute; diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index c9e02d85fb7..5cffe120925 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use proc_macro2::{Span, TokenStream}; use quote::ToTokens; use syn::{punctuated::Punctuated, spanned::Spanned, Token}; diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index 387934310b9..37c7e6e9b99 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! This crate declares only the proc macro attributes, as a crate defining proc macro attributes //! must not contain any other public items. diff --git a/src/callback.rs b/src/callback.rs index 6d59253730e..611b1787478 100644 --- a/src/callback.rs +++ b/src/callback.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Utilities for a Python callable object that invokes a Rust function. use crate::err::{PyErr, PyResult}; diff --git a/src/conversion.rs b/src/conversion.rs index c5bda16fd7c..85e9f04eb63 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Defines conversions between Rust and Python types. use crate::err::{self, PyDowncastError, PyResult}; #[cfg(feature = "experimental-inspect")] diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 08cc0fb11d5..52ff2149813 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - #![cfg(feature = "num-bigint")] //! Conversions to and from [num-bigint](https://docs.rs/num-bigint)’s [`BigInt`] and [`BigUint`] types. //! diff --git a/src/derive_utils.rs b/src/derive_utils.rs index 25ea7f89fa7..4ccb38f901b 100644 --- a/src/derive_utils.rs +++ b/src/derive_utils.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - //! Functionality for the code generated by the derive backend use crate::{types::PyModule, Python}; diff --git a/src/err/mod.rs b/src/err/mod.rs index 88b03986159..9c71b439352 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::panic::PanicException; use crate::type_object::PyTypeInfo; use crate::types::{PyTraceback, PyType}; diff --git a/src/exceptions.rs b/src/exceptions.rs index d92e8b35dc3..9cbc8587fe2 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Exception and warning types defined by Python. //! //! The structs in this module represent Python's built-in exceptions and diff --git a/src/gil.rs b/src/gil.rs index fa96a910797..342c8d9288f 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Interaction with Python's global interpreter lock use crate::impl_::not_send::{NotSend, NOT_SEND}; diff --git a/src/impl_/freelist.rs b/src/impl_/freelist.rs index d5e3d1f8143..955a50a3549 100644 --- a/src/impl_/freelist.rs +++ b/src/impl_/freelist.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Support for [free allocation lists][1]. //! //! This can improve performance for types that are often created and deleted in quick succession. diff --git a/src/instance.rs b/src/instance.rs index d3927768ec9..0cbf8dbfc68 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::conversion::PyTryFrom; use crate::err::{self, PyDowncastError, PyErr, PyResult}; use crate::gil; diff --git a/src/marker.rs b/src/marker.rs index c6d6fc40f96..2e6cbe599a5 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - //! Fundamental properties of objects tied to the Python interpreter. //! //! The Python interpreter is not threadsafe. To protect the Python interpreter in multithreaded diff --git a/src/prelude.rs b/src/prelude.rs index a110ea5c1f5..ca0b0cf38db 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! PyO3's prelude. //! //! The purpose of this module is to alleviate imports of many commonly used items of the PyO3 crate diff --git a/src/type_object.rs b/src/type_object.rs index f36abd94788..3098b07217f 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! Python type object information use crate::types::{PyAny, PyType}; diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 80c453a6522..919810d2ddc 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 927c450085c..c11af6d71f8 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::err::{PyErr, PyResult}; use crate::{ffi, AsPyPointer, Py, PyAny, Python}; use std::os::raw::c_char; diff --git a/src/types/capsule.rs b/src/types/capsule.rs index 7da59e735e3..152ac0ffc61 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::Python; use crate::{ffi, AsPyPointer, PyAny}; use crate::{pyobject_native_type_core, PyErr, PyResult}; diff --git a/src/types/code.rs b/src/types/code.rs index fc7e3e9f83e..c0d0ce83926 100644 --- a/src/types/code.rs +++ b/src/types/code.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2022-present PyO3 Project and Contributors - use crate::ffi; use crate::PyAny; diff --git a/src/types/dict.rs b/src/types/dict.rs index 0cf7e6ca7b6..0aedb13a76b 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use super::PyMapping; use crate::err::{self, PyErr, PyResult}; use crate::ffi::Py_ssize_t; diff --git a/src/types/floatob.rs b/src/types/floatob.rs index ccf73dd8cde..82db228dfa1 100644 --- a/src/types/floatob.rs +++ b/src/types/floatob.rs @@ -1,6 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ diff --git a/src/types/frame.rs b/src/types/frame.rs index c16e143987d..160f3b3b6ed 100644 --- a/src/types/frame.rs +++ b/src/types/frame.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2022-present PyO3 Project and Contributors - use crate::ffi; use crate::PyAny; diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 5b728784a30..fd2e51aa895 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -1,6 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// - #[cfg(Py_LIMITED_API)] use crate::types::PyIterator; use crate::{ diff --git a/src/types/iterator.rs b/src/types/iterator.rs index cfbbd31b6dc..3b45c11378a 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::{ffi, AsPyPointer, IntoPyPointer, Py, PyAny, PyErr, PyNativeType, PyResult, Python}; use crate::{PyDowncastError, PyTryFrom}; diff --git a/src/types/list.rs b/src/types/list.rs index 3e801276b95..5e951c29c8e 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use std::convert::TryInto; use crate::err::{self, PyResult}; diff --git a/src/types/mapping.rs b/src/types/mapping.rs index 602302a953b..fa73176d142 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::err::{PyDowncastError, PyErr, PyResult}; use crate::sync::GILOnceCell; use crate::type_object::PyTypeInfo; diff --git a/src/types/mod.rs b/src/types/mod.rs index 385d0af68dc..802a20c6f4a 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Various types defined by the Python interpreter such as `int`, `str` and `tuple`. pub use self::any::PyAny; diff --git a/src/types/module.rs b/src/types/module.rs index 970a22ff0ce..f728d7825d4 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::callback::IntoPyCallbackOutput; use crate::err::{PyErr, PyResult}; use crate::exceptions; diff --git a/src/types/num.rs b/src/types/num.rs index 21f6d72d3cd..522517155f8 100644 --- a/src/types/num.rs +++ b/src/types/num.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::{ffi, PyAny}; /// Represents a Python `int` object. diff --git a/src/types/sequence.rs b/src/types/sequence.rs index ccc2d895b72..73e110cd40e 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::err::{self, PyDowncastError, PyErr, PyResult}; use crate::exceptions::PyTypeError; #[cfg(feature = "experimental-inspect")] diff --git a/src/types/set.rs b/src/types/set.rs index 36774b9a014..cf043d99dae 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -1,6 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// - #[cfg(Py_LIMITED_API)] use crate::types::PyIterator; use crate::{ diff --git a/src/types/slice.rs b/src/types/slice.rs index 61574f6490e..e82b535a45b 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::err::{PyErr, PyResult}; use crate::ffi::{self, Py_ssize_t}; use crate::{AsPyPointer, PyAny, PyObject, Python, ToPyObject}; diff --git a/src/types/string.rs b/src/types/string.rs index 3998a6adf04..05b7109f280 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - #[cfg(not(Py_LIMITED_API))] use crate::exceptions::PyUnicodeDecodeError; use crate::types::PyBytes; diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 7e4393eb98d..82916558a55 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::err::{error_on_minusone, PyResult}; use crate::ffi; use crate::types::PyString; diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 3a124d3690e..429f2d0f848 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::convert::TryInto; use crate::ffi::{self, Py_ssize_t}; diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 00ba62c3be1..ca4a5cdbef6 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::err::{self, PyResult}; use crate::{ffi, AsPyPointer, PyAny, PyTypeInfo, Python}; From f28c280fbf1066e55cd6b77f2f81c59e801e8765 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 4 Jun 2023 22:58:44 +0100 Subject: [PATCH 06/59] run cargo check instead of clippy for MSRV in CI --- .github/workflows/ci.yml | 40 +++++++--- noxfile.py | 139 +++++++++++++++++---------------- pyo3-build-config/src/lib.rs | 5 ++ tests/test_class_attributes.rs | 2 +- 4 files changed, 106 insertions(+), 80 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 122d2007a12..b29e9d45655 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,32 @@ jobs: - name: Check rust formatting (rustfmt) run: nox -s fmt-rust + check-msrv: + needs: [fmt] + runs-on: ubuntu-latest + if: github.ref != 'refs/heads/main' + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.48.0 + targets: x86_64-unknown-linux-gnu + components: clippy,rust-src + - uses: actions/setup-python@v4 + with: + architecture: "x64" + - uses: Swatinem/rust-cache@v2 + with: + key: check-msrv-1.48.0 + continue-on-error: true + - run: python -m pip install --upgrade pip && pip install nox + - name: Prepare minimal package versions + run: nox -s set-minimal-package-versions + - run: nox -s check-all + + env: + CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu + clippy: needs: [fmt] runs-on: ${{ matrix.platform.os }} @@ -80,16 +106,6 @@ jobs: rust-target: "i686-pc-windows-msvc", }, ] - include: - - rust: 1.48.0 - python-version: "3.11" - platform: - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "x86_64-unknown-linux-gnu", - } - msrv: "MSRV" name: clippy/${{ matrix.platform.rust-target }}/${{ matrix.rust }} steps: - uses: actions/checkout@v3 @@ -106,9 +122,6 @@ jobs: key: clippy-${{ matrix.platform.rust-target }}-${{ matrix.platform.os }}-${{ matrix.rust }} continue-on-error: true - run: python -m pip install --upgrade pip && pip install nox - - if: matrix.msrv == 'MSRV' - name: Prepare minimal package versions (MSRV only) - run: nox -s set-minimal-package-versions - run: nox -s clippy-all env: CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }} @@ -353,6 +366,7 @@ jobs: conclusion: needs: - fmt + - check-msrv - clippy - build-pr - build-full diff --git a/noxfile.py b/noxfile.py index d264cdc0467..7b8246e1581 100644 --- a/noxfile.py +++ b/noxfile.py @@ -8,7 +8,7 @@ from functools import lru_cache from glob import glob from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple import nox @@ -96,25 +96,19 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: success = True env = env or os.environ for feature_set in _get_feature_sets(): - command = "clippy" - extra = ("--", "--deny=warnings") - if _get_rust_version()[:2] == (1, 48): - # 1.48 crashes during clippy because of lints requested - # in .cargo/config - command = "check" - extra = () try: _run( session, "cargo", - command, + "clippy", *feature_set, "--all-targets", "--workspace", # linting pyo3-ffi-check requires docs to have been built or # the macros will error; doesn't seem worth it on CI "--exclude=pyo3-ffi-check", - *extra, + "--", + "--deny=warnings", external=True, env=env, ) @@ -126,31 +120,43 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: @nox.session(name="clippy-all", venv_backend="none") def clippy_all(session: nox.Session) -> None: success = True - with tempfile.NamedTemporaryFile("r+") as config: - env = os.environ.copy() - env["PYO3_CONFIG_FILE"] = config.name - env["PYO3_CI"] = "1" - def _clippy_with_config(implementation, version) -> bool: - config.seek(0) - config.truncate(0) - config.write( - f"""\ -implementation={implementation} -version={version} -suppress_build_script_link_lines=true -""" - ) - config.flush() + def _clippy_with_config(env: Dict[str, str]) -> None: + nonlocal success + success &= _clippy(session, env=env) - session.log(f"{implementation} {version}") - return _clippy(session, env=env) + _for_all_version_configs(session, _clippy_with_config) - for version in PY_VERSIONS: - success &= _clippy_with_config("CPython", version) + if not success: + session.error("one or more jobs failed") - for version in PYPY_VERSIONS: - success &= _clippy_with_config("PyPy", version) + +@nox.session(name="check-all", venv_backend="none") +def check_all(session: nox.Session) -> None: + success = True + + def _check(env: Dict[str, str]) -> None: + nonlocal success + env = env or os.environ + for feature_set in _get_feature_sets(): + try: + _run( + session, + "cargo", + "check", + *feature_set, + "--all-targets", + "--workspace", + # linting pyo3-ffi-check requires docs to have been built or + # the macros will error; doesn't seem worth it on CI + "--exclude=pyo3-ffi-check", + external=True, + env=env, + ) + except Exception: + success = False + + _for_all_version_configs(session, _check) if not success: session.error("one or more jobs failed") @@ -422,42 +428,13 @@ def set_minimal_package_versions(session: nox.Session): "examples/word-count", ) min_pkg_versions = { - # newer versions of rust_decimal want newer arrayvec - "rust_decimal": "1.18.0", - # newer versions of arrayvec use const generics (Rust 1.51+) - "arrayvec": "0.5.2", + "rust_decimal": "1.26.1", "csv": "1.1.6", - # newer versions of chrono use i32::rem_euclid as a const fn - "chrono": "0.4.24", - "indexmap": "1.6.2", - "inventory": "0.3.4", - "hashbrown": "0.9.1", - "plotters": "0.3.1", - "plotters-svg": "0.3.1", - "plotters-backend": "0.3.2", - "bumpalo": "3.10.0", - "once_cell": "1.14.0", - "rayon": "1.5.3", - "rayon-core": "1.9.3", + "hashbrown": "0.12.3", + "once_cell": "1.17.2", + "rayon": "1.6.1", + "rayon-core": "1.10.2", "regex": "1.7.3", - # string_cache 0.8.4 depends on parking_lot 0.12 - "string_cache": "0.8.3", - # 1.15.0 depends on hermit-abi 0.2.6 which has edition 2021 and breaks 1.48.0 - "num_cpus": "1.14.0", - "parking_lot": "0.11.0", - # 1.0.77 needs basic-toml which has edition 2021 - "trybuild": "1.0.76", - # pins to avoid syn 2.0 (which requires Rust 1.56) - "ghost": "0.1.8", - "serde": "1.0.156", - "serde_derive": "1.0.156", - "cxx": "1.0.92", - "cxxbridge-macro": "1.0.92", - "cxx-build": "1.0.92", - "web-sys": "0.3.61", - "js-sys": "0.3.61", - "wasm-bindgen": "0.2.84", - "syn": "1.0.109", } # run cargo update first to ensure that everything is at highest @@ -634,7 +611,7 @@ def _run_cargo_set_package_version( *, project: Optional[str] = None, ) -> None: - command = ["cargo", "update", "-p", pkg_id, "--precise", version] + command = ["cargo", "update", "-p", pkg_id, "--precise", version, "--workspace"] if project: command.append(f"--manifest-path={project}/Cargo.toml") _run(session, *command, external=True) @@ -642,3 +619,33 @@ def _run_cargo_set_package_version( def _get_output(*args: str) -> str: return subprocess.run(args, capture_output=True, text=True, check=True).stdout + + +def _for_all_version_configs( + session: nox.Session, job: Callable[[Dict[str, str]], None] +) -> None: + with tempfile.NamedTemporaryFile("r+") as config: + env = os.environ.copy() + env["PYO3_CONFIG_FILE"] = config.name + env["PYO3_CI"] = "1" + + def _job_with_config(implementation, version) -> bool: + config.seek(0) + config.truncate(0) + config.write( + f"""\ +implementation={implementation} +version={version} +suppress_build_script_link_lines=true +""" + ) + config.flush() + + session.log(f"{implementation} {version}") + return job(env) + + for version in PY_VERSIONS: + _job_with_config("CPython", version) + + for version in PYPY_VERSIONS: + _job_with_config("PyPy", version) diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index f870daa7ea0..07c5bcb6d37 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -161,6 +161,11 @@ pub fn print_feature_cfgs() { if rustc_minor_version >= 59 { println!("cargo:rustc-cfg=thread_local_const_init"); } + + // Enable use of `#[cfg(panic = "...")]` on Rust 1.60 and greater + if rustc_minor_version >= 60 { + println!("cargo:rustc-cfg=panic_unwind"); + } } /// Private exports used in PyO3's build.rs diff --git a/tests/test_class_attributes.rs b/tests/test_class_attributes.rs index 043971455b6..e07aa457de7 100644 --- a/tests/test_class_attributes.rs +++ b/tests/test_class_attributes.rs @@ -96,7 +96,7 @@ fn recursive_class_attributes() { } #[test] -#[cfg(panic = "unwind")] +#[cfg_attr(cfg_panic, cfg(panic = "unwind"))] fn test_fallible_class_attribute() { use pyo3::{exceptions::PyValueError, types::PyString}; From d18e659edffbc9c9a31efb807baba23d0fbcfe11 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Mon, 5 Jun 2023 12:36:54 +0200 Subject: [PATCH 07/59] Drop usage of rustversion from xtask. --- xtask/Cargo.toml | 1 - xtask/src/cli.rs | 4 ++-- xtask/src/utils.rs | 35 ++--------------------------------- 3 files changed, 4 insertions(+), 36 deletions(-) diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 48a5a63fd3f..bdd52175f55 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -11,6 +11,5 @@ name = "xtask" anyhow = "1.0.51" # Clap 3 requires MSRV 1.54 -rustversion = "1.0" structopt = { version = "0.3", default-features = false } clap = { version = "2" } diff --git a/xtask/src/cli.rs b/xtask/src/cli.rs index f873816a017..2d146c9e7aa 100644 --- a/xtask/src/cli.rs +++ b/xtask/src/cli.rs @@ -107,7 +107,7 @@ impl Subcommand { /// Run a command as a child process, inheriting stdin, stdout and stderr. pub fn run(command: &mut Command) -> Result<()> { - let command_str = format_command(command); + let command_str = format!("{:?}", command); let github_actions = std::env::var_os("GITHUB_ACTIONS").is_some(); if github_actions { println!("::group::Running: {}", command_str); @@ -135,7 +135,7 @@ pub fn run(command: &mut Command) -> Result<()> { /// Like `run`, but does not inherit stdin, stdout and stderr. pub fn run_with_output(command: &mut Command) -> Result { - let command_str = format_command(command); + let command_str = format!("{:?}", command); println!("Running: {}", command_str); diff --git a/xtask/src/utils.rs b/xtask/src/utils.rs index 045697e7357..b694126375e 100644 --- a/xtask/src/utils.rs +++ b/xtask/src/utils.rs @@ -1,47 +1,16 @@ use anyhow::ensure; use std::process::Command; -// Replacement for str.split_once() on Rust older than 1.52 -#[rustversion::before(1.52)] -pub fn split_once(s: &str, pat: char) -> Option<(&str, &str)> { - let mut iter = s.splitn(2, pat); - Some((iter.next()?, iter.next()?)) -} - -#[rustversion::since(1.52)] -pub fn split_once(s: &str, pat: char) -> Option<(&str, &str)> { - s.split_once(pat) -} - -#[rustversion::since(1.57)] -pub fn format_command(command: &Command) -> String { - let mut buf = String::new(); - buf.push('`'); - buf.push_str(&command.get_program().to_string_lossy()); - for arg in command.get_args() { - buf.push(' '); - buf.push_str(&arg.to_string_lossy()); - } - buf.push('`'); - buf -} - -#[rustversion::before(1.57)] -pub fn format_command(command: &Command) -> String { - // Debug impl isn't as nice as the above, but will do on < 1.57 - format!("{:?}", command) -} - pub fn get_output(command: &mut Command) -> anyhow::Result { let output = command.output()?; ensure! { output.status.success(), - "process did not run successfully ({exit}): {command}", + "process did not run successfully ({exit}): {command:?}", exit = match output.status.code() { Some(code) => format!("exit code {}", code), None => "terminated by signal".into(), }, - command = format_command(command), + command = command, }; Ok(output) } From 96d026b27bfbacdc44bb3bbbf0b322ca4eeb48f4 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Mon, 5 Jun 2023 16:52:35 +0200 Subject: [PATCH 08/59] Remove sleeps between publishing as cargo will do that itself by now. --- noxfile.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/noxfile.py b/noxfile.py index 7b8246e1581..801011b9c10 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,7 +4,6 @@ import subprocess import sys import tempfile -import time from functools import lru_cache from glob import glob from pathlib import Path @@ -165,13 +164,9 @@ def _check(env: Dict[str, str]) -> None: @nox.session(venv_backend="none") def publish(session: nox.Session) -> None: _run_cargo_publish(session, package="pyo3-build-config") - time.sleep(10) _run_cargo_publish(session, package="pyo3-macros-backend") - time.sleep(10) _run_cargo_publish(session, package="pyo3-macros") - time.sleep(10) _run_cargo_publish(session, package="pyo3-ffi") - time.sleep(10) _run_cargo_publish(session, package="pyo3") From 0c541a6fea06998523a1c36c8f7d161ec40d68ec Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Mon, 5 Jun 2023 17:09:47 +0200 Subject: [PATCH 09/59] Add Nox session to build docs providing a replace for doc xtask subcommand. --- noxfile.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/noxfile.py b/noxfile.py index 801011b9c10..cf038db3fd0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -288,6 +288,45 @@ def test_emscripten(session: nox.Session): ) +@nox.session(venv_backend="none") +def docs(session: nox.Session) -> None: + rustdoc_flags = ["-Dwarnings"] + toolchain_flags = [] + cargo_flags = [] + + if "open" in session.posargs: + cargo_flags.append("--open") + + if "nightly" in session.posargs: + rustdoc_flags.append("--cfg docsrs") + toolchain_flags.append("+nightly") + cargo_flags.extend(["-Z", "unstable-options", "-Z", "rustdoc-scrape-examples"]) + + if "nightly" in session.posargs and "internal" in session.posargs: + rustdoc_flags.append("--Z unstable-options") + rustdoc_flags.append("--document-hidden-items") + cargo_flags.append("--document-private-items") + else: + cargo_flags.extend(["--exclude=pyo3-macros", "--exclude=pyo3-macros-backend"]) + + rustdoc_flags.append(session.env.get("RUSTDOCFLAGS", "")) + session.env["RUSTDOCFLAGS"] = " ".join(rustdoc_flags) + + _run( + session, + "cargo", + *toolchain_flags, + "doc", + "--lib", + "--no-default-features", + "--features=full", + "--no-deps", + "--workspace", + *cargo_flags, + external=True, + ) + + @nox.session(name="build-guide", venv_backend="none") def build_guide(session: nox.Session): _run(session, "mdbook", "build", "-d", "../target/guide", "guide", *session.posargs) From da0166993e425ee3b6e6bb21ce4f73a0059bde19 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Mon, 5 Jun 2023 17:12:30 +0200 Subject: [PATCH 10/59] Drop the xtask helper as it is superseded by Nox. While the xtask code base is better engineered than our slightly messy Nox manifest, all functionality is available via Nox including some that is not available via xtask. Finally, only the Nox sessions are used in the CI by now so that xtask does not see regular automated usage. --- .cargo/config | 3 - .netlify/build.sh | 4 +- Cargo.toml | 1 - Contributing.md | 10 +- xtask/Cargo.toml | 15 --- xtask/README.md | 23 ----- xtask/src/cli.rs | 216 ------------------------------------------- xtask/src/clippy.rs | 25 ----- xtask/src/doc.rs | 42 --------- xtask/src/fmt.rs | 23 ----- xtask/src/main.rs | 23 ----- xtask/src/pytests.rs | 27 ------ xtask/src/test.rs | 81 ---------------- xtask/src/utils.rs | 34 ------- 14 files changed, 8 insertions(+), 519 deletions(-) delete mode 100644 xtask/Cargo.toml delete mode 100644 xtask/README.md delete mode 100644 xtask/src/cli.rs delete mode 100644 xtask/src/clippy.rs delete mode 100644 xtask/src/doc.rs delete mode 100644 xtask/src/fmt.rs delete mode 100644 xtask/src/main.rs delete mode 100644 xtask/src/pytests.rs delete mode 100644 xtask/src/test.rs delete mode 100644 xtask/src/utils.rs diff --git a/.cargo/config b/.cargo/config index c17c0f23439..e87eb0f23b1 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,6 +1,3 @@ -[alias] -xtask = "run --package xtask --" - [target.'cfg(feature = "cargo-clippy")'] rustflags = [ # TODO: remove these allows once msrv increased from 1.48 diff --git a/.netlify/build.sh b/.netlify/build.sh index 74c8bca1927..215ee8c8c6d 100755 --- a/.netlify/build.sh +++ b/.netlify/build.sh @@ -53,7 +53,7 @@ mv target/guide netlify_build/main/ ## Build public docs -cargo xtask doc +nox -s docs mv target/doc netlify_build/main/doc/ echo "" > netlify_build/main/doc/index.html @@ -61,7 +61,7 @@ echo "" > netlify_build/main/doc/in ## Build internal docs echo "
⚠️ Internal Docs ⚠️ Not Public API 👉 Official Docs Here
" > netlify_build/banner.html -RUSTDOCFLAGS="--html-before-content netlify_build/banner.html" cargo xtask doc --internal +RUSTDOCFLAGS="--html-before-content netlify_build/banner.html" nox -s docs -- nightly internal rm netlify_build/banner.html mkdir -p netlify_build/internal diff --git a/Cargo.toml b/Cargo.toml index b0d84aae0d2..e8d4eda2f02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -183,7 +183,6 @@ members = [ "pyo3-macros-backend", "pytests", "examples", - "xtask" ] [package.metadata.docs.rs] diff --git a/Contributing.md b/Contributing.md index a1fd31a34d9..c6d29e2d427 100644 --- a/Contributing.md +++ b/Contributing.md @@ -51,7 +51,7 @@ There are some specific areas of focus where help is currently needed for the do You can build the docs (including all features) with ```shell -cargo xtask doc --open +nox -s docs -- open ``` #### Doctests @@ -95,8 +95,10 @@ Tests run with all supported Python versions with the latest stable Rust compile If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI. You can run these tests yourself with -```cargo xtask ci``` -See [its documentation](https://github.com/PyO3/pyo3/tree/main/xtask#readme) for more commands you can run. +```nox``` +and +```nox -l``` +lists further commands you can run. ### Documenting changes @@ -145,7 +147,7 @@ You can view what code is and isn't covered by PyO3's tests. We aim to have 100% - First, generate a `lcov.info` file with ```shell -cargo xtask coverage +nox -s coverage ``` You can install an IDE plugin to view the coverage. For example, if you use VSCode: - Add the [coverage-gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) plugin. diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml deleted file mode 100644 index bdd52175f55..00000000000 --- a/xtask/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "xtask" -version = "0.1.0" -edition = "2018" -publish = false - -[[bin]] -name = "xtask" - -[dependencies] -anyhow = "1.0.51" - -# Clap 3 requires MSRV 1.54 -structopt = { version = "0.3", default-features = false } -clap = { version = "2" } diff --git a/xtask/README.md b/xtask/README.md deleted file mode 100644 index 68d078e5595..00000000000 --- a/xtask/README.md +++ /dev/null @@ -1,23 +0,0 @@ -## Commands to test PyO3. - -To run these commands, you should be in PyO3's root directory, and run (for example) `cargo xtask ci`. - -``` -USAGE: - xtask.exe - -FLAGS: - -h, --help Prints help information - -V, --version Prints version information - -SUBCOMMANDS: - ci Runs everything - clippy Runs `clippy`, denying all warnings - coverage Runs `cargo llvm-cov` for the PyO3 codebase - default Only runs the fast things (this is used if no command is specified) - doc Attempts to render the documentation - fmt Checks Rust and Python code formatting with `rustfmt` and `black` - help Prints this message or the help of the given subcommand(s) - test Runs various variations on `cargo test` - test-py Runs the tests in examples/ and pytests/ -``` \ No newline at end of file diff --git a/xtask/src/cli.rs b/xtask/src/cli.rs deleted file mode 100644 index 2d146c9e7aa..00000000000 --- a/xtask/src/cli.rs +++ /dev/null @@ -1,216 +0,0 @@ -use crate::utils::*; -use anyhow::{ensure, Result}; -use std::io; -use std::process::{Command, Output}; -use std::time::Instant; -use structopt::StructOpt; - -pub const MSRV: &str = "1.48"; - -#[derive(StructOpt)] -pub enum Subcommand { - /// Only runs the fast things (this is used if no command is specified) - Default, - /// Runs everything - Ci, - /// Checks Rust and Python code formatting with `rustfmt` and `black` - Fmt, - /// Runs `clippy`, denying all warnings. - Clippy, - /// Attempts to render the documentation. - Doc(DocOpts), - /// Runs various variations on `cargo test` - Test, - /// Runs the tests in examples/ and pytests/ - TestPy, -} - -impl Default for Subcommand { - fn default() -> Self { - Self::Default - } -} - -#[derive(StructOpt)] -pub struct DocOpts { - /// Whether to run the docs using nightly rustdoc - #[structopt(long)] - pub stable: bool, - /// Whether to open the docs after rendering. - #[structopt(long)] - pub open: bool, - /// Whether to show the private and hidden API. - #[structopt(long)] - pub internal: bool, -} - -impl Default for DocOpts { - fn default() -> Self { - Self { - stable: true, - open: false, - internal: false, - } - } -} - -impl Subcommand { - pub fn execute(self) -> Result<()> { - print_metadata()?; - - let start = Instant::now(); - - match self { - Subcommand::Default => { - crate::fmt::rust::run()?; - crate::clippy::run()?; - crate::test::run()?; - crate::doc::run(DocOpts::default())?; - } - Subcommand::Ci => { - let installed = Installed::new()?; - crate::fmt::rust::run()?; - if installed.black { - crate::fmt::python::run()?; - } else { - Installed::warn_black() - }; - crate::clippy::run()?; - crate::test::run()?; - crate::doc::run(DocOpts::default())?; - if installed.nox { - crate::pytests::run(None)?; - } else { - Installed::warn_nox() - }; - installed.assert()? - } - - Subcommand::Doc(opts) => crate::doc::run(opts)?, - Subcommand::Fmt => { - crate::fmt::rust::run()?; - crate::fmt::python::run()?; - } - Subcommand::Clippy => crate::clippy::run()?, - Subcommand::TestPy => crate::pytests::run(None)?, - Subcommand::Test => crate::test::run()?, - }; - - let dt = start.elapsed().as_secs(); - let minutes = dt / 60; - let seconds = dt % 60; - println!("\nxtask finished in {}m {}s.", minutes, seconds); - - Ok(()) - } -} - -/// Run a command as a child process, inheriting stdin, stdout and stderr. -pub fn run(command: &mut Command) -> Result<()> { - let command_str = format!("{:?}", command); - let github_actions = std::env::var_os("GITHUB_ACTIONS").is_some(); - if github_actions { - println!("::group::Running: {}", command_str); - } else { - println!("Running: {}", command_str); - } - - let status = command.spawn()?.wait()?; - - ensure! { - status.success(), - "process did not run successfully ({exit}): {command}", - exit = match status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = command_str, - }; - - if github_actions { - println!("::endgroup::") - } - Ok(()) -} - -/// Like `run`, but does not inherit stdin, stdout and stderr. -pub fn run_with_output(command: &mut Command) -> Result { - let command_str = format!("{:?}", command); - - println!("Running: {}", command_str); - - let output = command.output()?; - - ensure! { - output.status.success(), - "process did not run successfully ({exit}): {command}:\n{stderr}", - exit = match output.status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = command_str, - stderr = String::from_utf8_lossy(&output.stderr) - }; - - Ok(output) -} - -#[derive(Copy, Clone, Debug)] -pub struct Installed { - pub nox: bool, - pub black: bool, -} - -impl Installed { - pub fn new() -> anyhow::Result { - Ok(Self { - nox: Self::nox()?, - black: Self::black()?, - }) - } - - pub fn nox() -> anyhow::Result { - let output = std::process::Command::new("nox").arg("--version").output(); - match output { - Ok(_) => Ok(true), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), - Err(other) => Err(other.into()), - } - } - - pub fn warn_nox() { - eprintln!("Skipping: formatting Python code, because `nox` was not found"); - } - - pub fn black() -> anyhow::Result { - let output = std::process::Command::new("black") - .arg("--version") - .output(); - match output { - Ok(_) => Ok(true), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), - Err(other) => Err(other.into()), - } - } - - pub fn warn_black() { - eprintln!("Skipping: Python code formatting, because `black` was not found."); - } - - pub fn assert(&self) -> anyhow::Result<()> { - if self.nox && self.black { - Ok(()) - } else { - let mut err = - String::from("\n\nxtask was unable to run all tests due to some missing programs:"); - if !self.black { - err.push_str("\n`black` was not installed. (`pip install black`)"); - } - if !self.nox { - err.push_str("\n`nox` was not installed. (`pip install nox`)"); - } - - Err(anyhow::anyhow!(err)) - } - } -} diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs deleted file mode 100644 index eec5f5fdd0b..00000000000 --- a/xtask/src/clippy.rs +++ /dev/null @@ -1,25 +0,0 @@ -use crate::cli; -use std::process::Command; - -pub fn run() -> anyhow::Result<()> { - cli::run( - Command::new("cargo") - .arg("clippy") - .arg("--features=full") - .arg("--all-targets") - .arg("--workspace") - .arg("--") - .arg("-Dwarnings"), - )?; - cli::run( - Command::new("cargo") - .arg("clippy") - .arg("--all-targets") - .arg("--workspace") - .arg("--features=abi3,full") - .arg("--") - .arg("-Dwarnings"), - )?; - - Ok(()) -} diff --git a/xtask/src/doc.rs b/xtask/src/doc.rs deleted file mode 100644 index cde2c882914..00000000000 --- a/xtask/src/doc.rs +++ /dev/null @@ -1,42 +0,0 @@ -use crate::cli; -use crate::cli::DocOpts; -use std::process::Command; -//--cfg docsrs --Z unstable-options --document-hidden-items - -pub fn run(opts: DocOpts) -> anyhow::Result<()> { - let mut flags = Vec::new(); - - if !opts.stable { - flags.push("--cfg docsrs"); - } - if opts.internal { - flags.push("--Z unstable-options"); - flags.push("--document-hidden-items"); - } - flags.push("-Dwarnings"); - - std::env::set_var("RUSTDOCFLAGS", flags.join(" ")); - cli::run( - Command::new(concat!(env!("CARGO_HOME"), "/bin/cargo")) - .args(if opts.stable { None } else { Some("+nightly") }) - .arg("doc") - .arg("--lib") - .arg("--no-default-features") - .arg("--features=full") - .arg("--no-deps") - .arg("--workspace") - .args(if opts.internal { - &["--document-private-items"][..] - } else { - &["--exclude=pyo3-macros", "--exclude=pyo3-macros-backend"][..] - }) - .args(if opts.stable { - &[][..] - } else { - &["-Z", "unstable-options", "-Z", "rustdoc-scrape-examples"][..] - }) - .args(if opts.open { Some("--open") } else { None }), - )?; - - Ok(()) -} diff --git a/xtask/src/fmt.rs b/xtask/src/fmt.rs deleted file mode 100644 index 8bc745246bc..00000000000 --- a/xtask/src/fmt.rs +++ /dev/null @@ -1,23 +0,0 @@ -pub mod rust { - use crate::cli; - use std::process::Command; - pub fn run() -> anyhow::Result<()> { - cli::run( - Command::new("cargo") - .arg("fmt") - .arg("--all") - .arg("--") - .arg("--check"), - )?; - Ok(()) - } -} - -pub mod python { - use crate::cli; - use std::process::Command; - pub fn run() -> anyhow::Result<()> { - cli::run(Command::new("black").arg(".").arg("--check"))?; - Ok(()) - } -} diff --git a/xtask/src/main.rs b/xtask/src/main.rs deleted file mode 100644 index 421742685ca..00000000000 --- a/xtask/src/main.rs +++ /dev/null @@ -1,23 +0,0 @@ -use clap::ErrorKind::MissingArgumentOrSubcommand; -use structopt::StructOpt; - -pub mod cli; -pub mod clippy; -pub mod doc; -pub mod fmt; -pub mod pytests; -pub mod test; -pub mod utils; - -fn main() -> anyhow::Result<()> { - // Avoid spewing backtraces all over the command line - // For some reason this is automatically enabled on nightly compilers... - std::env::set_var("RUST_LIB_BACKTRACE", "0"); - - match cli::Subcommand::from_args_safe() { - Ok(c) => c.execute()?, - Err(e) if e.kind == MissingArgumentOrSubcommand => cli::Subcommand::default().execute()?, - Err(e) => return Err(e.into()), - } - Ok(()) -} diff --git a/xtask/src/pytests.rs b/xtask/src/pytests.rs deleted file mode 100644 index 78744c69bd1..00000000000 --- a/xtask/src/pytests.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::cli; -use anyhow::Result; -use std::{path::Path, process::Command}; - -pub fn run<'a>(env: impl IntoIterator + Copy) -> Result<()> { - cli::run( - Command::new("nox") - .arg("--non-interactive") - .arg("-f") - .arg(Path::new("pytests").join("noxfile.py")) - .envs(env), - )?; - - for entry in std::fs::read_dir("examples")? { - let path = entry?.path(); - if path.is_dir() && path.join("noxfile.py").exists() { - cli::run( - Command::new("nox") - .arg("--non-interactive") - .arg("-f") - .arg(path.join("noxfile.py")) - .envs(env), - )?; - } - } - Ok(()) -} diff --git a/xtask/src/test.rs b/xtask/src/test.rs deleted file mode 100644 index b85f3e4a504..00000000000 --- a/xtask/src/test.rs +++ /dev/null @@ -1,81 +0,0 @@ -use crate::cli::{self, MSRV}; -use std::process::Command; - -pub fn run() -> anyhow::Result<()> { - cli::run( - Command::new("cargo") - .arg("test") - .arg("--lib") - .arg("--no-default-features") - .arg("--tests") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--no-default-features") - .arg("--features=full") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--no-default-features") - .arg("--features=abi3,full") - .arg("--quiet"), - )?; - - // If the MSRV toolchain is not installed, this will install it - cli::run( - Command::new("rustup") - .arg("toolchain") - .arg("install") - .arg(MSRV), - )?; - - // Test MSRV - cli::run( - Command::new(concat!(env!("CARGO_HOME"), "/bin/cargo")) - .arg(format!("+{}", MSRV)) - .arg("test") - .arg("--no-default-features") - .arg("--features=full,auto-initialize") - .arg("--quiet"), - )?; - - // If the nightly toolchain is not installed, this will install it - cli::run( - Command::new("rustup") - .arg("toolchain") - .arg("install") - .arg("nightly"), - )?; - - cli::run( - Command::new(concat!(env!("CARGO_HOME"), "/bin/cargo")) - .arg("+nightly") - .arg("test") - .arg("--no-default-features") - .arg("--features=full,nightly") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--manifest-path=pyo3-ffi/Cargo.toml") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--no-default-features") - .arg("--manifest-path=pyo3-build-config/Cargo.toml") - .arg("--quiet"), - )?; - - Ok(()) -} diff --git a/xtask/src/utils.rs b/xtask/src/utils.rs deleted file mode 100644 index b694126375e..00000000000 --- a/xtask/src/utils.rs +++ /dev/null @@ -1,34 +0,0 @@ -use anyhow::ensure; -use std::process::Command; - -pub fn get_output(command: &mut Command) -> anyhow::Result { - let output = command.output()?; - ensure! { - output.status.success(), - "process did not run successfully ({exit}): {command:?}", - exit = match output.status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = command, - }; - Ok(output) -} - -pub fn print_metadata() -> anyhow::Result<()> { - let rustc_output = std::process::Command::new("rustc") - .arg("--version") - .arg("--verbose") - .output()?; - let rustc_version = core::str::from_utf8(&rustc_output.stdout).unwrap(); - println!("Metadata: \n\n{}", rustc_version); - - let py_output = std::process::Command::new("python") - .arg("--version") - .arg("-V") - .output()?; - let py_version = core::str::from_utf8(&py_output.stdout).unwrap(); - println!("{}", py_version); - - Ok(()) -} From 8e42115e85f6279d66c020cf31e06379c256f004 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Mon, 5 Jun 2023 18:36:50 +0200 Subject: [PATCH 11/59] Add docs session to Nox's default session to more closely mirror the CI. --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index cf038db3fd0..ec3f62d8d8c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -11,7 +11,7 @@ import nox -nox.options.sessions = ["test", "clippy", "fmt"] +nox.options.sessions = ["test", "clippy", "fmt", "docs"] PYO3_DIR = Path(__file__).parent From b0279a766443848ffb8366ca80344eef17978960 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 4 Jun 2023 14:29:41 +0100 Subject: [PATCH 12/59] add github merge queue support --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b29e9d45655..279645bbbe8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,8 @@ on: # for bors try - trying pull_request: + merge_group: + types: [checks_requested] workflow_dispatch: concurrency: From 1315c164ed9e5cdab1d1a3f1d74011bb7d36c5db Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 6 Jun 2023 08:07:19 +0100 Subject: [PATCH 13/59] fix usage for paths-filter in merge_group --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2155b82683c..b8fa1f57346 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -125,6 +125,8 @@ jobs: if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy-3.7' && inputs.python-version != 'pypy-3.8' }} id: ffi-changes with: + base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} + ref: ${{ github.event.pull_request.head.ref || github.event.merge_group.head_ref }} filters: | changed: - 'pyo3-ffi/**' From 2dc5520e7066c3a38c1ada67f112d3057aa79c41 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 7 Jun 2023 08:22:11 +0100 Subject: [PATCH 14/59] ci: add pypy3.10-nightly --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 279645bbbe8..738b95795bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,9 +198,10 @@ jobs: "3.9", "3.10", "3.11", - "pypy-3.7", - "pypy-3.8", - "pypy-3.9" + "pypy3.7", + "pypy3.8", + "pypy3.9", + "pypy3.10-nightly", ] platform: [ From 29d303d032e61e9d08245859cecc8cb5ffaf164f Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 7 Jun 2023 21:05:13 +0100 Subject: [PATCH 15/59] readme: add media June 2023 --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cbfe3306e25..d6e3e75d728 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,9 @@ about this topic. ## Articles and other media -- [Making Python 100x faster with less than 100 lines of Rust](https://ohadravid.github.io/posts/2023-03-rusty-python/) - March 28, 2023 +- [A Week of PyO3 + rust-numpy (How to Speed Up Your Data Pipeline X Times)](https://terencezl.github.io/blog/2023/06/06/a-week-of-pyo3-rust-numpy/) - Jun 6, 2023 +- [(Podcast) PyO3 with David Hewitt](https://rustacean-station.org/episode/david-hewitt/) - May 19, 2023 +- [Making Python 100x faster with less than 100 lines of Rust](https://ohadravid.github.io/posts/2023-03-rusty-python/) - Mar 28, 2023 - [How Pydantic V2 leverages Rust's Superpowers](https://fosdem.org/2023/schedule/event/rust_how_pydantic_v2_leverages_rusts_superpowers/) - Feb 4, 2023 - [How we extended the River stats module with Rust using PyO3](https://boring-guy.sh/posts/river-rust/) - Dec 23, 2022 - [Nine Rules for Writing Python Extensions in Rust](https://towardsdatascience.com/nine-rules-for-writing-python-extensions-in-rust-d35ea3a4ec29?sk=f8d808d5f414154fdb811e4137011437) - Dec 31, 2021 From e545689969ce514169f6eeaeaa24fcc34ba832f1 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 7 Jun 2023 22:22:32 +0100 Subject: [PATCH 16/59] ci: enable full builds with a label --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 738b95795bc..c82e27c7e2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,7 +129,7 @@ jobs: CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }} build-pr: - if: github.event_name == 'pull_request' + if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-build-full') && github.event_name == 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} needs: [fmt] uses: ./.github/workflows/build.yml @@ -173,7 +173,7 @@ jobs: } ] build-full: - if: ${{ github.event_name != 'pull_request' && github.ref != 'refs/heads/main' }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} needs: [fmt] uses: ./.github/workflows/build.yml From f0b0d26ead465adfc202dc7d010f1f7b8d4754b0 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 8 Jun 2023 07:17:03 +0100 Subject: [PATCH 17/59] ci: fail ffi-check on pypy failures --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8fa1f57346..e905120f321 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -122,7 +122,7 @@ jobs: - uses: dorny/paths-filter@v2 # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here - if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy-3.7' && inputs.python-version != 'pypy-3.8' }} + if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' }} id: ffi-changes with: base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} @@ -136,9 +136,9 @@ jobs: - name: Run pyo3-ffi-check run: nox -s ffi-check - # Allow failure on PyPy for now - continue-on-error: ${{ startsWith(inputs.python-version, 'pypy') }} - if: ${{ steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy-3.7' && inputs.python-version != 'pypy-3.8' }} + # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor + # is pypy 3.9 on windows + if: ${{ steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows')) }} - name: Test cross compilation From 0cd1a52f0eb6414872b8e1a2ade320fa2b584705 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 8 Jun 2023 07:48:15 +0100 Subject: [PATCH 18/59] fix pypy 3.10 ffi definitions --- pyo3-ffi/src/cpython/object.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 870473b46da..7f39a16cb3a 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -276,9 +276,9 @@ pub struct PyTypeObject { pub tp_finalize: Option, #[cfg(Py_3_8)] pub tp_vectorcall: Option, - #[cfg(any(all(PyPy, Py_3_8), all(not(PyPy), Py_3_8, not(Py_3_9))))] + #[cfg(any(all(PyPy, Py_3_8, not(Py_3_10)), all(not(PyPy), Py_3_8, not(Py_3_9))))] pub tp_print: Option, - #[cfg(PyPy)] + #[cfg(all(PyPy, not(Py_3_10)))] pub tp_pypy_flags: std::os::raw::c_long, #[cfg(py_sys_config = "COUNT_ALLOCS")] pub tp_allocs: Py_ssize_t, From a96d7e7fc0f6f3710fc73474bc9fe50b60590089 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 8 Jun 2023 20:56:40 +0100 Subject: [PATCH 19/59] ci: remove bors configuration --- .github/bors.toml | 7 ------- .github/workflows/ci.yml | 4 ---- 2 files changed, 11 deletions(-) delete mode 100644 .github/bors.toml diff --git a/.github/bors.toml b/.github/bors.toml deleted file mode 100644 index dfe28aba8d9..00000000000 --- a/.github/bors.toml +++ /dev/null @@ -1,7 +0,0 @@ -delete_merged_branches = true -required_approvals = 0 -use_codeowners = false -status = [ - "conclusion", -] -timeout_sec = 21600 \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c82e27c7e2b..5b8f9f51f96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,10 +4,6 @@ on: push: branches: - main - # for bors r+ - - staging - # for bors try - - trying pull_request: merge_group: types: [checks_requested] From 3983de7fa3c7dd41a2cab5b171d93d15924fec40 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 11 Jun 2023 20:52:37 +0100 Subject: [PATCH 20/59] ci: pool can be dirty from other threads --- src/gil.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/gil.rs b/src/gil.rs index 342c8d9288f..ad4716d98c1 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -487,7 +487,7 @@ mod tests { use crate::{ffi, gil, AsPyPointer, IntoPyPointer, PyObject, Python, ToPyObject}; #[cfg(not(target_arch = "wasm32"))] use parking_lot::{const_mutex, Condvar, Mutex}; - use std::{ptr::NonNull, sync::atomic::Ordering}; + use std::ptr::NonNull; fn get_object(py: Python<'_>) -> PyObject { // Convenience function for getting a single unique object, using `new_pool` so as to leave @@ -503,8 +503,20 @@ mod tests { OWNED_OBJECTS.with(|holder| holder.borrow().len()) } - fn pool_not_dirty() -> bool { - !POOL.dirty.load(Ordering::SeqCst) + fn pool_inc_refs_does_not_contain(obj: &PyObject) -> bool { + !POOL + .pointer_ops + .lock() + .0 + .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) + } + + fn pool_dec_refs_does_not_contain(obj: &PyObject) -> bool { + !POOL + .pointer_ops + .lock() + .1 + .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } #[cfg(not(target_arch = "wasm32"))] @@ -582,13 +594,13 @@ mod tests { let reference = obj.clone_ref(py); assert_eq!(obj.get_refcnt(py), 2); - assert!(pool_not_dirty()); + assert!(pool_inc_refs_does_not_contain(&obj)); // With the GIL held, reference cound will be decreased immediately. drop(reference); assert_eq!(obj.get_refcnt(py), 1); - assert!(pool_not_dirty()); + assert!(pool_dec_refs_does_not_contain(&obj)); }); } @@ -601,7 +613,7 @@ mod tests { let reference = obj.clone_ref(py); assert_eq!(obj.get_refcnt(py), 2); - assert!(pool_not_dirty()); + assert!(pool_inc_refs_does_not_contain(&obj)); // Drop reference in a separate thread which doesn't have the GIL. std::thread::spawn(move || drop(reference)).join().unwrap(); From 91a0166e53a268815fb59bbd98d88c30b1b15be9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 1 Jun 2023 13:29:23 +0200 Subject: [PATCH 21/59] Add `IpAddr` <-> `ipaddress.IPv(4/6)Address` conversion --- newsfragments/3197.added.md | 1 + src/conversions/std/ipaddr.rs | 110 ++++++++++++++++++++++++++++++++++ src/conversions/std/mod.rs | 1 + src/sync.rs | 17 +++++- 4 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3197.added.md create mode 100755 src/conversions/std/ipaddr.rs diff --git a/newsfragments/3197.added.md b/newsfragments/3197.added.md new file mode 100644 index 00000000000..eb8affb1042 --- /dev/null +++ b/newsfragments/3197.added.md @@ -0,0 +1 @@ +Add support for converting to and from Python's `ipaddress.IPv4Address`/`ipaddress.IPv6Address` and `std::net::IpAddr`. diff --git a/src/conversions/std/ipaddr.rs b/src/conversions/std/ipaddr.rs new file mode 100755 index 00000000000..ca3c8728f9b --- /dev/null +++ b/src/conversions/std/ipaddr.rs @@ -0,0 +1,110 @@ +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +use crate::exceptions::PyValueError; +use crate::sync::GILOnceCell; +use crate::types::PyType; +use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; + +impl FromPyObject<'_> for IpAddr { + fn extract(obj: &PyAny) -> PyResult { + match obj.getattr(intern!(obj.py(), "packed")) { + Ok(packed) => { + if let Ok(packed) = packed.extract::<[u8; 4]>() { + Ok(IpAddr::V4(Ipv4Addr::from(packed))) + } else if let Ok(packed) = packed.extract::<[u8; 16]>() { + Ok(IpAddr::V6(Ipv6Addr::from(packed))) + } else { + Err(PyValueError::new_err("invalid packed length")) + } + } + Err(_) => { + // We don't have a .packed attribute, so we try to construct an IP from str(). + obj.str()?.to_str()?.parse().map_err(PyValueError::new_err) + } + } + } +} + +impl ToPyObject for Ipv4Addr { + fn to_object(&self, py: Python<'_>) -> PyObject { + static IPV4_ADDRESS: GILOnceCell> = GILOnceCell::new(); + IPV4_ADDRESS + .get_or_try_init_type_ref(py, "ipaddress", "IPv4Address") + .expect("failed to load ipaddress.IPv4Address") + .call1((u32::from_be_bytes(self.octets()),)) + .expect("failed to construct ipaddress.IPv4Address") + .to_object(py) + } +} + +impl ToPyObject for Ipv6Addr { + fn to_object(&self, py: Python<'_>) -> PyObject { + static IPV6_ADDRESS: GILOnceCell> = GILOnceCell::new(); + IPV6_ADDRESS + .get_or_try_init_type_ref(py, "ipaddress", "IPv6Address") + .expect("failed to load ipaddress.IPv6Address") + .call1((u128::from_be_bytes(self.octets()),)) + .expect("failed to construct ipaddress.IPv6Address") + .to_object(py) + } +} + +impl ToPyObject for IpAddr { + fn to_object(&self, py: Python<'_>) -> PyObject { + match self { + IpAddr::V4(ip) => ip.to_object(py), + IpAddr::V6(ip) => ip.to_object(py), + } + } +} + +impl IntoPy for IpAddr { + fn into_py(self, py: Python<'_>) -> PyObject { + self.to_object(py) + } +} + +#[cfg(test)] +mod test_ipaddr { + use std::str::FromStr; + + use crate::types::PyString; + + use super::*; + + #[test] + fn test_roundtrip() { + Python::with_gil(|py| { + fn roundtrip(py: Python<'_>, ip: &str) { + let ip = IpAddr::from_str(ip).unwrap(); + let py_cls = if ip.is_ipv4() { + "IPv4Address" + } else { + "IPv6Address" + }; + + let pyobj = ip.into_py(py); + let repr = pyobj.as_ref(py).repr().unwrap().to_string_lossy(); + assert_eq!(repr, format!("{}('{}')", py_cls, ip)); + + let ip2: IpAddr = pyobj.extract(py).unwrap(); + assert_eq!(ip, ip2); + } + roundtrip(py, "127.0.0.1"); + roundtrip(py, "::1"); + roundtrip(py, "0.0.0.0"); + }); + } + + #[test] + fn test_from_pystring() { + Python::with_gil(|py| { + let py_str = PyString::new(py, "0:0:0:0:0:0:0:1"); + let ip: IpAddr = py_str.to_object(py).extract(py).unwrap(); + assert_eq!(ip, IpAddr::from_str("::1").unwrap()); + + let py_str = PyString::new(py, "invalid"); + assert!(py_str.to_object(py).extract::(py).is_err()); + }); + } +} diff --git a/src/conversions/std/mod.rs b/src/conversions/std/mod.rs index 6021c395288..f5e917d08ea 100644 --- a/src/conversions/std/mod.rs +++ b/src/conversions/std/mod.rs @@ -1,4 +1,5 @@ mod array; +mod ipaddr; mod map; mod num; mod osstr; diff --git a/src/sync.rs b/src/sync.rs index 0f5a51631d3..3cb4206d239 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,5 +1,5 @@ //! Synchronization mechanisms based on the Python GIL. -use crate::{types::PyString, Py, Python}; +use crate::{types::PyString, types::PyType, Py, PyErr, Python}; use std::cell::UnsafeCell; /// Value with concurrent access protected by the GIL. @@ -169,6 +169,21 @@ impl GILOnceCell { } } +impl GILOnceCell> { + /// Get a reference to the contained Python type, initializing it if needed. + /// + /// This is a shorthand method for `get_or_init` which imports the type from Python on init. + pub(crate) fn get_or_try_init_type_ref<'py>( + &'py self, + py: Python<'py>, + module_name: &str, + attr_name: &str, + ) -> Result<&'py PyType, PyErr> { + self.get_or_try_init(py, || py.import(module_name)?.getattr(attr_name)?.extract()) + .map(|ty| ty.as_ref(py)) + } +} + /// Interns `text` as a Python string and stores a reference to it in static storage. /// /// A reference to the same Python string is returned on each invocation. From f775ca9a8647bdcd6ae1ea792a95a661405bcef4 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 22 Nov 2022 22:19:12 +0000 Subject: [PATCH 22/59] ci: add Python 3.12-dev jobs --- .github/workflows/ci.yml | 1 + pyo3-ffi-check/src/main.rs | 4 ++-- pyo3-ffi/src/cpython/code.rs | 25 ++++++++++++++++++++++++- pyo3-ffi/src/cpython/compile.rs | 15 +++++++++++++++ pyo3-ffi/src/cpython/initconfig.rs | 6 +++++- pyo3-ffi/src/cpython/object.rs | 2 ++ pyo3-ffi/src/cpython/pyerrors.rs | 2 ++ pyo3-ffi/src/cpython/unicodeobject.rs | 26 +++++++++++++++++--------- pyo3-ffi/src/unicodeobject.rs | 2 ++ src/ffi/tests.rs | 11 ++++++++++- 10 files changed, 80 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b8f9f51f96..b4fcce39c46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,6 +194,7 @@ jobs: "3.9", "3.10", "3.11", + "3.12-dev", "pypy3.7", "pypy3.8", "pypy3.9", diff --git a/pyo3-ffi-check/src/main.rs b/pyo3-ffi-check/src/main.rs index c537362530b..36af5c160a4 100644 --- a/pyo3-ffi-check/src/main.rs +++ b/pyo3-ffi-check/src/main.rs @@ -40,9 +40,9 @@ fn main() { pyo3_ffi_align, bindgen_align ); - } else { - pyo3_ffi_check_macro::for_all_fields!($name, check_field); } + + pyo3_ffi_check_macro::for_all_fields!($name, check_field); }}; } diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index b77b2f988d0..fa5b1c20df9 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -2,7 +2,7 @@ use crate::object::*; use crate::pyport::Py_ssize_t; #[allow(unused_imports)] -use std::os::raw::{c_char, c_int, c_uchar, c_void}; +use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void}; // skipped _Py_CODEUNIT // skipped _Py_OPCODE @@ -11,6 +11,16 @@ use std::os::raw::{c_char, c_int, c_uchar, c_void}; #[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))] opaque_struct!(_PyOpcache); +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCoCached { + pub _co_code: *mut PyObject, + pub _co_varnames: *mut PyObject, + pub _co_cellvars: *mut PyObject, + pub _co_freevars: *mut PyObject, +} + #[cfg(all(not(PyPy), not(Py_3_7)))] opaque_struct!(PyCodeObject); @@ -83,17 +93,24 @@ pub struct PyCodeObject { pub co_names: *mut PyObject, pub co_exceptiontable: *mut PyObject, pub co_flags: c_int, + #[cfg(not(Py_3_12))] pub co_warmup: c_int, + #[cfg(Py_3_12)] + pub _co_linearray_entry_size: c_short, pub co_argcount: c_int, pub co_posonlyargcount: c_int, pub co_kwonlyargcount: c_int, pub co_stacksize: c_int, pub co_firstlineno: c_int, + pub co_nlocalsplus: c_int, + #[cfg(Py_3_12)] + pub co_framesize: c_int, pub co_nlocals: c_int, pub co_nplaincellvars: c_int, pub co_ncellvars: c_int, pub co_nfreevars: c_int, + pub co_localsplusnames: *mut PyObject, pub co_localspluskinds: *mut PyObject, pub co_filename: *mut PyObject, @@ -101,9 +118,15 @@ pub struct PyCodeObject { pub co_qualname: *mut PyObject, pub co_linetable: *mut PyObject, pub co_weakreflist: *mut PyObject, + #[cfg(not(Py_3_12))] pub _co_code: *mut PyObject, + #[cfg(Py_3_12)] + pub _co_cached: *mut _PyCoCached, + #[cfg(not(Py_3_12))] pub _co_linearray: *mut c_char, pub _co_firsttraceable: c_int, + #[cfg(Py_3_12)] + pub _co_linearray: *mut c_char, pub co_extra: *mut c_void, pub co_code_adaptive: [c_char; 1], } diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 9a2afdb93e3..71af81e83e5 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -30,12 +30,27 @@ pub struct PyCompilerFlags { // skipped non-limited _PyCompilerFlags_INIT +#[cfg(all(Py_3_12, not(PyPy)))] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCompilerSrcLocation { + pub lineno: c_int, + pub end_lineno: c_int, + pub col_offset: c_int, + pub end_col_offset: c_int, +} + +// skipped SRC_LOCATION_FROM_AST + #[cfg(not(PyPy))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyFutureFeatures { pub ff_features: c_int, + #[cfg(not(Py_3_12))] pub ff_lineno: c_int, + #[cfg(Py_3_12)] + pub ff_location: _PyCompilerSrcLocation, } pub const FUTURE_NESTED_SCOPES: &str = "nested_scopes"; diff --git a/pyo3-ffi/src/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs index 7d40ba08d38..17fe7559e1b 100644 --- a/pyo3-ffi/src/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -91,6 +91,8 @@ pub struct PyConfig { #[cfg(all(Py_3_9, not(Py_3_10)))] pub _use_peg_parser: c_int, pub tracemalloc: c_int, + #[cfg(Py_3_12)] + pub perf_profiling: c_int, pub import_time: c_int, #[cfg(Py_3_11)] pub code_debug_ranges: c_int, @@ -137,6 +139,8 @@ pub struct PyConfig { pub use_frozen_modules: c_int, #[cfg(Py_3_11)] pub safe_path: c_int, + #[cfg(Py_3_12)] + pub int_max_str_digits: c_int, pub pathconfig_warnings: c_int, #[cfg(Py_3_10)] pub program_name: *mut wchar_t, @@ -163,7 +167,7 @@ pub struct PyConfig { pub run_filename: *mut wchar_t, pub _install_importlib: c_int, pub _init_main: c_int, - #[cfg(Py_3_9)] + #[cfg(all(Py_3_9, not(Py_3_12)))] pub _isolated_interpreter: c_int, #[cfg(Py_3_11)] pub _is_python_build: c_int, diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 7f39a16cb3a..76ab074f3d0 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -276,6 +276,8 @@ pub struct PyTypeObject { pub tp_finalize: Option, #[cfg(Py_3_8)] pub tp_vectorcall: Option, + #[cfg(Py_3_12)] + pub tp_watched: c_char, #[cfg(any(all(PyPy, Py_3_8, not(Py_3_10)), all(not(PyPy), Py_3_8, not(Py_3_9))))] pub tp_print: Option, #[cfg(all(PyPy, not(Py_3_10)))] diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs index abbffc0b8c2..fe7b4d4b045 100644 --- a/pyo3-ffi/src/cpython/pyerrors.rs +++ b/pyo3-ffi/src/cpython/pyerrors.rs @@ -65,6 +65,8 @@ pub struct PyImportErrorObject { pub msg: *mut PyObject, pub name: *mut PyObject, pub path: *mut PyObject, + #[cfg(Py_3_12)] + pub name_from: *mut PyObject, } #[cfg(not(PyPy))] diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index 7cd3bfbc8fa..dc5d3ff92f4 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -118,10 +118,7 @@ where } const STATE_INTERNED_INDEX: usize = 0; -#[cfg(not(Py_3_12))] const STATE_INTERNED_WIDTH: u8 = 2; -#[cfg(Py_3_12)] -const STATE_INTERNED_WIDTH: u8 = 1; const STATE_KIND_INDEX: usize = STATE_INTERNED_WIDTH as usize; const STATE_KIND_WIDTH: u8 = 3; @@ -265,20 +262,22 @@ pub struct PyASCIIObject { /// Interacting with the bitfield is not actually well-defined, so we mark these APIs unsafe. impl PyASCIIObject { + #[cfg_attr(not(Py_3_12), allow(rustdoc::broken_intra_doc_links))] // SSTATE_INTERNED_IMMORTAL_STATIC requires 3.12 /// Get the `interned` field of the [`PyASCIIObject`] state bitfield. /// /// Returns one of: [`SSTATE_NOT_INTERNED`], [`SSTATE_INTERNED_MORTAL`], - /// or on CPython earlier than 3.12, [`SSTATE_INTERNED_IMMORTAL`] + /// [`SSTATE_INTERNED_IMMORTAL`], or [`SSTATE_INTERNED_IMMORTAL_STATIC`]. #[inline] pub unsafe fn interned(&self) -> c_uint { PyASCIIObjectState::from(self.state).interned() } + #[cfg_attr(not(Py_3_12), allow(rustdoc::broken_intra_doc_links))] // SSTATE_INTERNED_IMMORTAL_STATIC requires 3.12 /// Set the `interned` field of the [`PyASCIIObject`] state bitfield. /// /// Calling this function with an argument that is not [`SSTATE_NOT_INTERNED`], - /// [`SSTATE_INTERNED_MORTAL`], or on CPython earlier than 3.12, - /// [`SSTATE_INTERNED_IMMORTAL`] is invalid. + /// [`SSTATE_INTERNED_MORTAL`], [`SSTATE_INTERNED_IMMORTAL`], or + /// [`SSTATE_INTERNED_IMMORTAL_STATIC`] is invalid. #[inline] pub unsafe fn set_interned(&mut self, val: c_uint) { let mut state = PyASCIIObjectState::from(self.state); @@ -398,12 +397,14 @@ extern "C" { pub const SSTATE_NOT_INTERNED: c_uint = 0; pub const SSTATE_INTERNED_MORTAL: c_uint = 1; -#[cfg(not(Py_3_12))] pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2; +#[cfg(Py_3_12)] +pub const SSTATE_INTERNED_IMMORTAL_STATIC: c_uint = 3; #[inline] pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); + #[cfg(not(Py_3_12))] debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).ascii() @@ -420,7 +421,7 @@ pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint { } #[cfg(not(Py_3_12))] -#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))] +#[deprecated(note = "Removed in Python 3.12")] pub const PyUnicode_WCHAR_KIND: c_uint = 0; pub const PyUnicode_1BYTE_KIND: c_uint = 1; @@ -445,6 +446,7 @@ pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { #[inline] pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); + #[cfg(not(Py_3_12))] debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).kind() @@ -484,6 +486,7 @@ pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { #[inline] pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t { debug_assert!(crate::PyUnicode_Check(op) != 0); + #[cfg(not(Py_3_12))] debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).length @@ -502,8 +505,13 @@ pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).ready() } +#[cfg(Py_3_12)] +#[inline] +pub unsafe fn PyUnicode_READY(_op: *mut PyObject) -> c_int { + 0 +} + #[cfg(not(Py_3_12))] -#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))] #[inline] pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int { debug_assert!(crate::PyUnicode_Check(op) != 0); diff --git a/pyo3-ffi/src/unicodeobject.rs b/pyo3-ffi/src/unicodeobject.rs index 86075475475..5ce6496834c 100644 --- a/pyo3-ffi/src/unicodeobject.rs +++ b/pyo3-ffi/src/unicodeobject.rs @@ -59,6 +59,8 @@ extern "C" { pub fn PyUnicode_AsUCS4Copy(unicode: *mut PyObject) -> *mut Py_UCS4; #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetLength")] pub fn PyUnicode_GetLength(unicode: *mut PyObject) -> Py_ssize_t; + #[cfg(not(Py_3_12))] + #[deprecated(note = "Removed in Python 3.12")] #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetSize")] pub fn PyUnicode_GetSize(unicode: *mut PyObject) -> Py_ssize_t; pub fn PyUnicode_ReadChar(unicode: *mut PyObject, index: Py_ssize_t) -> Py_UCS4; diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index 97e838cf527..68ddab76305 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -2,6 +2,7 @@ use crate::ffi::*; use crate::{types::PyDict, AsPyPointer, IntoPy, Py, PyAny, Python}; use crate::types::PyString; +#[cfg(not(Py_3_12))] use libc::wchar_t; #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons @@ -116,6 +117,7 @@ fn ascii_object_bitfield() { #[cfg(not(PyPy))] hash: 0, state: 0u32, + #[cfg(not(Py_3_12))] wstr: std::ptr::null_mut() as *mut wchar_t, }; @@ -124,9 +126,12 @@ fn ascii_object_bitfield() { assert_eq!(o.kind(), 0); assert_eq!(o.compact(), 0); assert_eq!(o.ascii(), 0); + #[cfg(not(Py_3_12))] assert_eq!(o.ready(), 0); - for i in 0..4 { + let interned_count = if cfg!(Py_3_12) { 2 } else { 4 }; + + for i in 0..interned_count { o.set_interned(i); assert_eq!(o.interned(), i); } @@ -142,7 +147,9 @@ fn ascii_object_bitfield() { o.set_ascii(1); assert_eq!(o.ascii(), 1); + #[cfg(not(Py_3_12))] o.set_ready(1); + #[cfg(not(Py_3_12))] assert_eq!(o.ready(), 1); } } @@ -163,6 +170,7 @@ fn ascii() { assert_eq!(ascii.kind(), PyUnicode_1BYTE_KIND); assert_eq!(ascii.compact(), 1); assert_eq!(ascii.ascii(), 1); + #[cfg(not(Py_3_12))] assert_eq!(ascii.ready(), 1); assert_eq!(PyUnicode_IS_ASCII(ptr), 1); @@ -203,6 +211,7 @@ fn ucs4() { assert_eq!(ascii.kind(), PyUnicode_4BYTE_KIND); assert_eq!(ascii.compact(), 1); assert_eq!(ascii.ascii(), 0); + #[cfg(not(Py_3_12))] assert_eq!(ascii.ready(), 1); assert_eq!(PyUnicode_IS_ASCII(ptr), 0); From 098df7b2c884e55bf1e0d55e2b3e62a29cf88371 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 8 Jun 2023 07:28:14 +0100 Subject: [PATCH 23/59] ci: allow failures in 3.12 builds for now --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e905120f321..df448c7d4bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ on: jobs: build: + continue-on-error: ${{ inputs.python-version == '3.12-dev' }} runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v3 From a38f923f38cf7641879f00a0ba58e91f2a2a8d53 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 12 Jun 2023 20:30:03 +0100 Subject: [PATCH 24/59] ci: allow failure on 3.7 and pypy nightly --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df448c7d4bd..a003feffc13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ on: jobs: build: - continue-on-error: ${{ inputs.python-version == '3.12-dev' }} + continue-on-error: ${{ contains(fromJSON('["3.7", "3.12-dev", "pypy3.7", "pypy3.10-nightly"]'), inputs.python-version) }} runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v3 From 196c3dfc0e55b180029f2529666f6250a9da4bed Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:18:12 +0100 Subject: [PATCH 25/59] rename examples package to fix dependabot --- examples/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Cargo.toml b/examples/Cargo.toml index e76132c4afe..43907a8becd 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "examples" +name = "pyo3-examples" version = "0.0.0" publish = false edition = "2018" From 906a85ce016f97fba78bc86c8fe7e9637f057bd3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Jun 2023 07:39:44 +0000 Subject: [PATCH 26/59] Update bindgen requirement from 0.63.0 to 0.65.1 Updates the requirements on [bindgen](https://github.com/rust-lang/rust-bindgen) to permit the latest version. - [Changelog](https://github.com/rust-lang/rust-bindgen/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-lang/rust-bindgen/compare/v0.63.0...v0.65.1) --- updated-dependencies: - dependency-name: bindgen dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pyo3-ffi-check/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyo3-ffi-check/Cargo.toml b/pyo3-ffi-check/Cargo.toml index e1663eb8cdc..9957bef8bf2 100644 --- a/pyo3-ffi-check/Cargo.toml +++ b/pyo3-ffi-check/Cargo.toml @@ -13,5 +13,5 @@ path = "../pyo3-ffi" features = ["extension-module"] # A lazy way of skipping linking in most cases (as we don't use any runtime symbols) [build-dependencies] -bindgen = "0.63.0" +bindgen = "0.65.1" pyo3-build-config = { path = "../pyo3-build-config" } From 796b84861bb448fb94b4df921f6d846b8bb5aa3b Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 13 Jun 2023 15:59:35 -0500 Subject: [PATCH 27/59] Add `PyFrozenSetBuilder` --- newsfragments/3156.added.md | 1 + src/types/frozenset.rs | 53 +++++++++++++++++++++++++++++++++++++ src/types/mod.rs | 2 +- 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3156.added.md diff --git a/newsfragments/3156.added.md b/newsfragments/3156.added.md new file mode 100644 index 00000000000..75a83ef0fe8 --- /dev/null +++ b/newsfragments/3156.added.md @@ -0,0 +1 @@ +Add `pyo3::types::PyFrozenSetBuilder` to allow building a `PyFrozenSet` item by item. \ No newline at end of file diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index fd2e51aa895..ecfbbb430c5 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -8,6 +8,38 @@ use crate::{ffi, AsPyPointer, PyAny, Python, ToPyObject}; use std::ptr; +/// Allows building a Python `frozenset` one item at a time +pub struct PyFrozenSetBuilder<'py> { + py_frozen_set: &'py PyFrozenSet, +} + +impl<'py> PyFrozenSetBuilder<'py> { + /// Create a new `FrozenSetBuilder`. + /// Since this allocates a `PyFrozenSet` internally it may + /// panic when running out of memory. + pub fn new(py: Python<'py>) -> PyResult> { + Ok(PyFrozenSetBuilder { + py_frozen_set: PyFrozenSet::empty(py)?, + }) + } + + /// Adds an element to the set. + pub fn add(&mut self, key: K) -> PyResult<()> + where + K: ToPyObject, + { + let py = self.py_frozen_set.py(); + err::error_on_minusone(py, unsafe { + ffi::PySet_Add(self.py_frozen_set.as_ptr(), key.to_object(py).as_ptr()) + }) + } + + /// Finish building the set and take ownership of its current value + pub fn finalize(self) -> &'py PyFrozenSet { + self.py_frozen_set + } +} + /// Represents a Python `frozenset` #[repr(transparent)] pub struct PyFrozenSet(PyAny); @@ -235,4 +267,25 @@ mod tests { } }); } + + #[test] + fn test_frozenset_builder() { + use super::PyFrozenSetBuilder; + + Python::with_gil(|py| { + let mut builder = PyFrozenSetBuilder::new(py).unwrap(); + + // add an item + builder.add(1).unwrap(); + builder.add(2).unwrap(); + builder.add(2).unwrap(); + + // finalize it + let set = builder.finalize(); + + assert!(set.contains(1).unwrap()); + assert!(set.contains(2).unwrap()); + assert!(!set.contains(3).unwrap()); + }); + } } diff --git a/src/types/mod.rs b/src/types/mod.rs index 802a20c6f4a..06a677ab62d 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -19,7 +19,7 @@ pub use self::dict::{PyDictItems, PyDictKeys, PyDictValues}; pub use self::floatob::PyFloat; #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] pub use self::frame::PyFrame; -pub use self::frozenset::PyFrozenSet; +pub use self::frozenset::{PyFrozenSet, PyFrozenSetBuilder}; pub use self::function::PyCFunction; #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] pub use self::function::PyFunction; From 6c86afe7721465711b0f1964d3cd321c7425858a Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 14 Jun 2023 08:45:32 +0100 Subject: [PATCH 28/59] bump maturin version in example files --- examples/decorator/.template/pyproject.toml | 2 +- examples/decorator/pyproject.toml | 2 +- examples/maturin-starter/.template/pyproject.toml | 2 +- examples/maturin-starter/pyproject.toml | 2 +- examples/plugin/plugin_api/pyproject.toml | 2 +- examples/word-count/pyproject.toml | 2 +- guide/src/getting_started.md | 2 +- pytests/pyproject.toml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/decorator/.template/pyproject.toml b/examples/decorator/.template/pyproject.toml index cd79e887b23..537fdacc666 100644 --- a/examples/decorator/.template/pyproject.toml +++ b/examples/decorator/.template/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/decorator/pyproject.toml b/examples/decorator/pyproject.toml index c6196256d35..8575ca25fc2 100644 --- a/examples/decorator/pyproject.toml +++ b/examples/decorator/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/maturin-starter/.template/pyproject.toml b/examples/maturin-starter/.template/pyproject.toml index cd79e887b23..537fdacc666 100644 --- a/examples/maturin-starter/.template/pyproject.toml +++ b/examples/maturin-starter/.template/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/maturin-starter/pyproject.toml b/examples/maturin-starter/pyproject.toml index 9a18a20ea8b..fb9c808f283 100644 --- a/examples/maturin-starter/pyproject.toml +++ b/examples/maturin-starter/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/plugin/plugin_api/pyproject.toml b/examples/plugin/plugin_api/pyproject.toml index 114687eddef..6350644ca14 100644 --- a/examples/plugin/plugin_api/pyproject.toml +++ b/examples/plugin/plugin_api/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.14,<0.15"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/word-count/pyproject.toml b/examples/word-count/pyproject.toml index d8ce3650277..6f88a5170f1 100644 --- a/examples/word-count/pyproject.toml +++ b/examples/word-count/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/guide/src/getting_started.md b/guide/src/getting_started.md index 2dd45a43ce7..2074590e325 100644 --- a/guide/src/getting_started.md +++ b/guide/src/getting_started.md @@ -123,7 +123,7 @@ You should also create a `pyproject.toml` with the following contents: ```toml [build-system] -requires = ["maturin>=0.14,<0.15"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index 9d2fb926c66..dfebfe31173 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [tool.pytest.ini_options] From 8f01440e2765f21aed9ba8f5644cc41cf6c59516 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Thu, 15 Jun 2023 20:34:32 +0200 Subject: [PATCH 29/59] Extend guide section on classes regarding free functions It might not be obvious from the reading the sections on free functions and on classes that they combine in a frictionless manner, i.e. class instances can be parameters to free functions in the same manner that the self parameters of instance methods are handled. This also explicitly calls out the interaction between `Clone` and `FromPyObject` for classes. --- guide/src/class.md | 73 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index dae5cbc04ec..4b1b7c55165 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -226,8 +226,10 @@ struct FrozenCounter { value: AtomicUsize, } -let py_counter: Py = Python::with_gil(|py| { - let counter = FrozenCounter { value: AtomicUsize::new(0) }; +let py_counter: Py = Python::with_gil(|py| { + let counter = FrozenCounter { + value: AtomicUsize::new(0), + }; Py::new(py, counter).unwrap() }); @@ -647,9 +649,9 @@ impl BaseClass { #[new] #[classmethod] fn py_new<'p>(cls: &'p PyType, py: Python<'p>) -> PyResult { - // Get an abstract attribute (presumably) declared on a subclass of this class. - let subclass_attr = cls.getattr("a_class_attr")?; - Ok(Self(subclass_attr.to_object(py))) + // Get an abstract attribute (presumably) declared on a subclass of this class. + let subclass_attr = cls.getattr("a_class_attr")?; + Ok(Self(subclass_attr.to_object(py))) } } ``` @@ -716,6 +718,67 @@ impl MyClass { } ``` +## Free functions + +Free functions defined using `#[pyfunction]` interact with classes through the same mechanisms as the self parameters of instance methods, i.e. they can take GIL-bound references, GIL-bound reference wrappers or GIL-indepedent references: + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyclass] +struct MyClass { + my_field: i32, +} + +// Take a GIL-bound reference when the underlying `PyCell` is irrelevant. +#[pyfunction] +fn increment_field(my_class: &mut MyClass) { + my_class.my_field += 1; +} + +// Take a GIL-bound reference wrapper when borrowing should be automatic, +// but interaction with the underlying `PyCell` is desired. +#[pyfunction] +fn print_field(my_class: PyRef<'_, MyClass>) { + println!("{}", my_class.my_field); +} + +// Take a GIL-bound reference to the underyling cell +// when borrowing needs to be managed manaually. +#[pyfunction] +fn increment_then_print_field(my_class: &PyCell) { + my_class.borrow_mut().my_field += 1; + + println!("{}", my_class.borrow().my_field); +} + +// Take a GIL-indepedent reference when you want to store the reference elsewhere. +#[pyfunction] +fn print_refcnt(my_class: Py, py: Python<'_>) { + println!("{}", my_class.get_refcnt(py)); +} +``` + +Classes can also be passed by value if they can be be cloned, i.e. they automatically implement `FromPyObject` if they implement `Clone`, e.g. via `#[derive(Clone)]`: + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyclass] +#[derive(Clone)] +struct MyClass { + my_field: Box, +} + +#[pyfunction] +fn dissamble_clone(my_class: MyClass) { + let MyClass { mut my_field } = my_class; + *my_field += 1; +} +``` + +Note that `#[derive(FromPyObject)]` on a class is usually not useful as it tries to construct a new Rust value by filling in the fields by looking up attributes of any given Python value. + ## Method arguments Similar to `#[pyfunction]`, the `#[pyo3(signature = (...))]` attribute can be used to specify the way that `#[pymethods]` accept arguments. Consult the documentation for [`function signatures`](./function/signature.md) to see the parameters this attribute accepts. From 605262f88ca9c143c516fd222628248d383793d5 Mon Sep 17 00:00:00 2001 From: Luca Trevisani Date: Fri, 16 Jun 2023 23:45:49 +0200 Subject: [PATCH 30/59] Improve `PyModule::from_code` examples --- src/types/module.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types/module.rs b/src/types/module.rs index f728d7825d4..b6b25eacb95 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -98,7 +98,7 @@ impl PyModule { /// let code = include_str!("../../assets/script.py"); /// /// Python::with_gil(|py| -> PyResult<()> { - /// PyModule::from_code(py, code, "example", "example")?; + /// PyModule::from_code(py, code, "example.py", "example")?; /// Ok(()) /// })?; /// # Ok(()) @@ -117,7 +117,7 @@ impl PyModule { /// let code = std::fs::read_to_string("assets/script.py")?; /// /// Python::with_gil(|py| -> PyResult<()> { - /// PyModule::from_code(py, &code, "example", "example")?; + /// PyModule::from_code(py, &code, "example.py", "example")?; /// Ok(()) /// })?; /// Ok(()) From 9ccca9f0eaa1523b0ca1557ebed875b44e591c72 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sat, 17 Jun 2023 13:27:46 +0200 Subject: [PATCH 31/59] Drop reference pool's dirty flag is the additional cost does not outweigh the improvement compared to locking an uncontented mutex. --- src/gil.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/gil.rs b/src/gil.rs index ad4716d98c1..7900f9ab4ab 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -4,7 +4,7 @@ use crate::impl_::not_send::{NotSend, NOT_SEND}; use crate::{ffi, Python}; use parking_lot::{const_mutex, Mutex, Once}; use std::cell::{Cell, RefCell}; -use std::{mem, ptr::NonNull, sync::atomic}; +use std::{mem, ptr::NonNull}; static START: Once = Once::new(); @@ -238,7 +238,6 @@ type PyObjVec = Vec>; /// Thread-safe storage for objects which were inc_ref / dec_ref while the GIL was not held. struct ReferencePool { - dirty: atomic::AtomicBool, // .0 is INCREFs, .1 is DECREFs pointer_ops: Mutex<(PyObjVec, PyObjVec)>, } @@ -246,30 +245,27 @@ struct ReferencePool { impl ReferencePool { const fn new() -> Self { Self { - dirty: atomic::AtomicBool::new(false), pointer_ops: const_mutex((Vec::new(), Vec::new())), } } fn register_incref(&self, obj: NonNull) { self.pointer_ops.lock().0.push(obj); - self.dirty.store(true, atomic::Ordering::Release); } fn register_decref(&self, obj: NonNull) { self.pointer_ops.lock().1.push(obj); - self.dirty.store(true, atomic::Ordering::Release); } fn update_counts(&self, _py: Python<'_>) { - let prev = self.dirty.swap(false, atomic::Ordering::Acquire); - if !prev { + let mut ops = self.pointer_ops.lock(); + if ops.0.is_empty() && ops.1.is_empty() { return; } - let mut ops = self.pointer_ops.lock(); let (increfs, decrefs) = mem::take(&mut *ops); drop(ops); + // Always increase reference counts first - as otherwise objects which have a // nonzero total reference count might be incorrectly dropped by Python during // this update. From e14961acf7c3805062c377f28add6ebe1a976a58 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sat, 17 Jun 2023 14:00:28 +0200 Subject: [PATCH 32/59] We already carefully handle re-entrancy for OWNED_OBJECTS, so let's avoid the runtime checking overhead. --- src/gil.rs | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/gil.rs b/src/gil.rs index 7900f9ab4ab..b128076e1e9 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -3,7 +3,7 @@ use crate::impl_::not_send::{NotSend, NOT_SEND}; use crate::{ffi, Python}; use parking_lot::{const_mutex, Mutex, Once}; -use std::cell::{Cell, RefCell}; +use std::cell::{Cell, UnsafeCell}; use std::{mem, ptr::NonNull}; static START: Once = Once::new(); @@ -33,7 +33,7 @@ thread_local_const_init! { static GIL_COUNT: Cell = const { Cell::new(0) }; /// Temporarily hold objects that will be released when the GILPool drops. - static OWNED_OBJECTS: RefCell>> = const { RefCell::new(Vec::new()) }; + static OWNED_OBJECTS: UnsafeCell = const { UnsafeCell::new(Vec::new()) }; } const GIL_LOCKED_DURING_TRAVERSE: isize = -1; @@ -373,7 +373,12 @@ impl GILPool { // Update counts of PyObjects / Py that have been cloned or dropped since last acquisition POOL.update_counts(Python::assume_gil_acquired()); GILPool { - start: OWNED_OBJECTS.try_with(|o| o.borrow().len()).ok(), + start: OWNED_OBJECTS + .try_with(|owned_objects| { + // SAFETY: This is not re-entrant. + unsafe { (*owned_objects.get()).len() } + }) + .ok(), _not_send: NOT_SEND, } } @@ -387,18 +392,18 @@ impl GILPool { impl Drop for GILPool { fn drop(&mut self) { - if let Some(obj_len_start) = self.start { - let dropping_obj = OWNED_OBJECTS.with(|holder| { - // `holder` must be dropped before calling Py_DECREF, or Py_DECREF may call - // `GILPool::drop` recursively, resulting in invalid borrowing. - let mut holder = holder.borrow_mut(); - if obj_len_start < holder.len() { - holder.split_off(obj_len_start) + if let Some(start) = self.start { + let owned_objects = OWNED_OBJECTS.with(|owned_objects| { + // SAFETY: `OWNED_OBJECTS` is released before calling Py_DECREF, + // or Py_DECREF may call `GILPool::drop` recursively, resulting in invalid borrowing. + let owned_objects = unsafe { &mut *owned_objects.get() }; + if start < owned_objects.len() { + owned_objects.split_off(start) } else { Vec::new() } }); - for obj in dropping_obj { + for obj in owned_objects { unsafe { ffi::Py_DECREF(obj.as_ptr()); } @@ -447,7 +452,12 @@ pub unsafe fn register_decref(obj: NonNull) { pub unsafe fn register_owned(_py: Python<'_>, obj: NonNull) { debug_assert!(gil_is_acquired()); // Ignores the error in case this function called from `atexit`. - let _ = OWNED_OBJECTS.try_with(|holder| holder.borrow_mut().push(obj)); + let _ = OWNED_OBJECTS.try_with(|owned_objects| { + // SAFETY: This is not re-entrant. + unsafe { + (*owned_objects.get()).push(obj); + } + }); } /// Increments pyo3's internal GIL count - to be called whenever GILPool or GILGuard is created. @@ -496,7 +506,7 @@ mod tests { } fn owned_object_count() -> usize { - OWNED_OBJECTS.with(|holder| holder.borrow().len()) + OWNED_OBJECTS.with(|owned_objects| unsafe { (*owned_objects.get()).len() }) } fn pool_inc_refs_does_not_contain(obj: &PyObject) -> bool { From bf2a193a439d5e768b2e66e923c6defb2f542fdb Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sun, 18 Jun 2023 15:25:37 +0200 Subject: [PATCH 33/59] Keep the dynamic borrow checking enabled for debug builds. --- src/gil.rs | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/gil.rs b/src/gil.rs index b128076e1e9..69b511c9ff3 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -3,7 +3,11 @@ use crate::impl_::not_send::{NotSend, NOT_SEND}; use crate::{ffi, Python}; use parking_lot::{const_mutex, Mutex, Once}; -use std::cell::{Cell, UnsafeCell}; +use std::cell::Cell; +#[cfg(debug_assertions)] +use std::cell::RefCell; +#[cfg(not(debug_assertions))] +use std::cell::UnsafeCell; use std::{mem, ptr::NonNull}; static START: Once = Once::new(); @@ -33,6 +37,9 @@ thread_local_const_init! { static GIL_COUNT: Cell = const { Cell::new(0) }; /// Temporarily hold objects that will be released when the GILPool drops. + #[cfg(debug_assertions)] + static OWNED_OBJECTS: RefCell = const { RefCell::new(Vec::new()) }; + #[cfg(not(debug_assertions))] static OWNED_OBJECTS: UnsafeCell = const { UnsafeCell::new(Vec::new()) }; } @@ -375,8 +382,12 @@ impl GILPool { GILPool { start: OWNED_OBJECTS .try_with(|owned_objects| { + #[cfg(debug_assertions)] + let len = owned_objects.borrow().len(); + #[cfg(not(debug_assertions))] // SAFETY: This is not re-entrant. - unsafe { (*owned_objects.get()).len() } + let len = unsafe { (*owned_objects.get()).len() }; + len }) .ok(), _not_send: NOT_SEND, @@ -394,6 +405,9 @@ impl Drop for GILPool { fn drop(&mut self) { if let Some(start) = self.start { let owned_objects = OWNED_OBJECTS.with(|owned_objects| { + #[cfg(debug_assertions)] + let mut owned_objects = owned_objects.borrow_mut(); + #[cfg(not(debug_assertions))] // SAFETY: `OWNED_OBJECTS` is released before calling Py_DECREF, // or Py_DECREF may call `GILPool::drop` recursively, resulting in invalid borrowing. let owned_objects = unsafe { &mut *owned_objects.get() }; @@ -453,6 +467,9 @@ pub unsafe fn register_owned(_py: Python<'_>, obj: NonNull) { debug_assert!(gil_is_acquired()); // Ignores the error in case this function called from `atexit`. let _ = OWNED_OBJECTS.try_with(|owned_objects| { + #[cfg(debug_assertions)] + owned_objects.borrow_mut().push(obj); + #[cfg(not(debug_assertions))] // SAFETY: This is not re-entrant. unsafe { (*owned_objects.get()).push(obj); @@ -506,7 +523,11 @@ mod tests { } fn owned_object_count() -> usize { - OWNED_OBJECTS.with(|owned_objects| unsafe { (*owned_objects.get()).len() }) + #[cfg(debug_assertions)] + let len = OWNED_OBJECTS.with(|owned_objects| owned_objects.borrow().len()); + #[cfg(not(debug_assertions))] + let len = OWNED_OBJECTS.with(|owned_objects| unsafe { (*owned_objects.get()).len() }); + len } fn pool_inc_refs_does_not_contain(obj: &PyObject) -> bool { From 44f308fd46d1f9ef664b6abdc6eb987f6c709210 Mon Sep 17 00:00:00 2001 From: ijl Date: Fri, 16 Jun 2023 14:49:22 +0000 Subject: [PATCH 34/59] Add PyErr_GetRaisedException(), PyErr_SetRaisedException() to FFI These are new and part of the stable API as of v3.12.0a6. PyErr_Fetch(), PyErr_Restore(), and PyErr_NormalizeException() were simultaneously deprecated in favor of these. python/cpython@feec49c40736fc05626a183a8d14c4ebbea5ae28 --- newsfragments/3248.added.md | 1 + pyo3-ffi/src/pyerrors.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 newsfragments/3248.added.md diff --git a/newsfragments/3248.added.md b/newsfragments/3248.added.md new file mode 100644 index 00000000000..5fb5656f753 --- /dev/null +++ b/newsfragments/3248.added.md @@ -0,0 +1 @@ +Add `PyErr_GetRaisedException()`, `PyErr_SetRaisedException()` to FFI definitions for Python 3.12 and later. diff --git a/pyo3-ffi/src/pyerrors.rs b/pyo3-ffi/src/pyerrors.rs index 91d7e378799..b80f009b982 100644 --- a/pyo3-ffi/src/pyerrors.rs +++ b/pyo3-ffi/src/pyerrors.rs @@ -13,12 +13,14 @@ extern "C" { pub fn PyErr_Occurred() -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyErr_Clear")] pub fn PyErr_Clear(); + #[cfg_attr(Py_3_12, deprecated(note = "Use PyErr_GetRaisedException() instead."))] #[cfg_attr(PyPy, link_name = "PyPyErr_Fetch")] pub fn PyErr_Fetch( arg1: *mut *mut PyObject, arg2: *mut *mut PyObject, arg3: *mut *mut PyObject, ); + #[cfg_attr(Py_3_12, deprecated(note = "Use PyErr_SetRaisedException() instead."))] #[cfg_attr(PyPy, link_name = "PyPyErr_Restore")] pub fn PyErr_Restore(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPyErr_GetExcInfo")] @@ -35,12 +37,22 @@ extern "C" { pub fn PyErr_GivenExceptionMatches(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyErr_ExceptionMatches")] pub fn PyErr_ExceptionMatches(arg1: *mut PyObject) -> c_int; + #[cfg_attr( + Py_3_12, + deprecated( + note = "Use PyErr_GetRaisedException() instead, to avoid any possible de-normalization." + ) + )] #[cfg_attr(PyPy, link_name = "PyPyErr_NormalizeException")] pub fn PyErr_NormalizeException( arg1: *mut *mut PyObject, arg2: *mut *mut PyObject, arg3: *mut *mut PyObject, ); + #[cfg(Py_3_12)] + pub fn PyErr_GetRaisedException() -> *mut PyObject; + #[cfg(Py_3_12)] + pub fn PyErr_SetRaisedException(exc: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPyException_SetTraceback")] pub fn PyException_SetTraceback(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyException_GetTraceback")] From 2934db1684551b4be285008d83a9c0f995dcd20e Mon Sep 17 00:00:00 2001 From: mejrs <59372212+mejrs@users.noreply.github.com> Date: Mon, 19 Jun 2023 19:48:04 +0200 Subject: [PATCH 35/59] Replace unreachable! with empty enum match --- pyo3-build-config/src/errors.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs index 6f57e0e6eed..26108c29564 100644 --- a/pyo3-build-config/src/errors.rs +++ b/pyo3-build-config/src/errors.rs @@ -87,8 +87,8 @@ impl From<&'_ str> for Error { } impl From for Error { - fn from(_: std::convert::Infallible) -> Self { - unreachable!() + fn from(x: std::convert::Infallible) -> Self { + match x {} } } From 4c232cbe763c0c79c2183dad90452afbd06f90ff Mon Sep 17 00:00:00 2001 From: mejrs <59372212+mejrs@users.noreply.github.com> Date: Mon, 19 Jun 2023 20:17:50 +0200 Subject: [PATCH 36/59] Use `value` as argument name --- pyo3-build-config/src/errors.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs index 26108c29564..8670b355bac 100644 --- a/pyo3-build-config/src/errors.rs +++ b/pyo3-build-config/src/errors.rs @@ -87,8 +87,8 @@ impl From<&'_ str> for Error { } impl From for Error { - fn from(x: std::convert::Infallible) -> Self { - match x {} + fn from(value: std::convert::Infallible) -> Self { + match value {} } } From 6d79007c4bc0e8ad6901e3fd7dc6f24aa345c2cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 20:58:37 +0000 Subject: [PATCH 37/59] Update hashbrown requirement from >= 0.9, < 0.14 to >= 0.9, < 0.15 Updates the requirements on [hashbrown](https://github.com/rust-lang/hashbrown) to permit the latest version. - [Changelog](https://github.com/rust-lang/hashbrown/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/hashbrown/compare/v0.9.0...v0.14.0) --- updated-dependencies: - dependency-name: hashbrown dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e8d4eda2f02..59b01d38000 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ inventory = { version = "0.3.0", optional = true } anyhow = { version = "1.0", optional = true } chrono = { version = "0.4", default-features = false, optional = true } eyre = { version = ">= 0.4, < 0.7", optional = true } -hashbrown = { version = ">= 0.9, < 0.14", optional = true } +hashbrown = { version = ">= 0.9, < 0.15", optional = true } indexmap = { version = "1.6", optional = true } num-bigint = { version = "0.4", optional = true } num-complex = { version = ">= 0.2, < 0.5", optional = true } From 94a47447253100b3a5c7d4fb2823f7334fe5804f Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Tue, 20 Jun 2023 19:38:43 +0200 Subject: [PATCH 38/59] Add changelog entry for bumping hashbrown dependency. --- newsfragments/3258.packaging.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3258.packaging.md diff --git a/newsfragments/3258.packaging.md b/newsfragments/3258.packaging.md new file mode 100644 index 00000000000..758e54584d9 --- /dev/null +++ b/newsfragments/3258.packaging.md @@ -0,0 +1 @@ +Enabled support for hashbrown version 0.14 From cd5724cf5595cc38d1b99a54f6cb35c8c53a4ec8 Mon Sep 17 00:00:00 2001 From: mejrs <59372212+mejrs@users.noreply.github.com> Date: Mon, 19 Jun 2023 23:25:51 +0200 Subject: [PATCH 39/59] Give a better error message for Python in traverse --- pyo3-macros-backend/src/pymethod.rs | 17 +++++++++++++---- tests/test_compile_error.rs | 2 +- tests/ui/traverse.rs | 27 +++++++++++++++++++++++++++ tests/ui/traverse.stderr | 23 +++++++++++++++++++++++ tests/ui/traverse_bare_self.rs | 12 ------------ tests/ui/traverse_bare_self.stderr | 17 ----------------- 6 files changed, 64 insertions(+), 34 deletions(-) create mode 100644 tests/ui/traverse.rs create mode 100644 tests/ui/traverse.stderr delete mode 100644 tests/ui/traverse_bare_self.rs delete mode 100644 tests/ui/traverse_bare_self.stderr diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 76ebc947028..38c325113f4 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -204,7 +204,7 @@ pub fn gen_py_method( GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec)?) } PyMethodProtoKind::Traverse => { - GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec.name)) + GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec)?) } PyMethodProtoKind::SlotFragment(slot_fragment_def) => { let proto = slot_fragment_def.generate_pyproto_fragment(cls, spec)?; @@ -398,7 +398,16 @@ fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>) -> Result MethodAndSlotDef { +fn impl_traverse_slot(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { + if let (Some(py_arg), _) = split_off_python_arg(&spec.signature.arguments) { + return Err(syn::Error::new_spanned(py_arg.ty, "__traverse__ may not take `Python`. \ + Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. \ + Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, \ + i.e. `Python::with_gil` will panic.")); + } + + let rust_fn_ident = spec.name; + let associated_method = quote! { pub unsafe extern "C" fn __pymethod_traverse__( slf: *mut _pyo3::ffi::PyObject, @@ -414,10 +423,10 @@ fn impl_traverse_slot(cls: &syn::Type, rust_fn_ident: &syn::Ident) -> MethodAndS pfunc: #cls::__pymethod_traverse__ as _pyo3::ffi::traverseproc as _ } }; - MethodAndSlotDef { + Ok(MethodAndSlotDef { associated_method, slot_def, - } + }) } fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 4625ea16431..5d933e13bc6 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -37,5 +37,5 @@ fn test_compile_errors() { t.compile_fail("tests/ui/not_send.rs"); t.compile_fail("tests/ui/not_send2.rs"); t.compile_fail("tests/ui/get_set_all.rs"); - t.compile_fail("tests/ui/traverse_bare_self.rs"); + t.compile_fail("tests/ui/traverse.rs"); } diff --git a/tests/ui/traverse.rs b/tests/ui/traverse.rs new file mode 100644 index 00000000000..034224951c9 --- /dev/null +++ b/tests/ui/traverse.rs @@ -0,0 +1,27 @@ +use pyo3::prelude::*; +use pyo3::PyVisit; +use pyo3::PyTraverseError; + +#[pyclass] +struct TraverseTriesToTakePyRef {} + +#[pymethods] +impl TraverseTriesToTakePyRef { + fn __traverse__(slf: PyRef, visit: PyVisit) {} +} + +#[pyclass] +struct Class; + +#[pymethods] +impl Class { + fn __traverse__(&self, py: Python<'_>, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + Ok(()) + } + + fn __clear__(&mut self) { + } +} + + +fn main() {} diff --git a/tests/ui/traverse.stderr b/tests/ui/traverse.stderr new file mode 100644 index 00000000000..873ef864a23 --- /dev/null +++ b/tests/ui/traverse.stderr @@ -0,0 +1,23 @@ +error: __traverse__ may not take `Python`. Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. + --> tests/ui/traverse.rs:18:32 + | +18 | fn __traverse__(&self, py: Python<'_>, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + | ^^^^^^^^^^ + +error[E0308]: mismatched types + --> tests/ui/traverse.rs:9:6 + | +8 | #[pymethods] + | ------------ arguments to this function are incorrect +9 | impl TraverseTriesToTakePyRef { + | ______^ +10 | | fn __traverse__(slf: PyRef, visit: PyVisit) {} + | |___________________^ expected fn pointer, found fn item + | + = note: expected fn pointer `for<'a, 'b> fn(&'a TraverseTriesToTakePyRef, PyVisit<'b>) -> Result<(), PyTraverseError>` + found fn item `for<'a, 'b> fn(pyo3::PyRef<'a, TraverseTriesToTakePyRef>, PyVisit<'b>) {TraverseTriesToTakePyRef::__traverse__}` +note: function defined here + --> src/impl_/pymethods.rs + | + | pub unsafe fn call_traverse_impl( + | ^^^^^^^^^^^^^^^^^^ diff --git a/tests/ui/traverse_bare_self.rs b/tests/ui/traverse_bare_self.rs deleted file mode 100644 index 5adc316e43f..00000000000 --- a/tests/ui/traverse_bare_self.rs +++ /dev/null @@ -1,12 +0,0 @@ -use pyo3::prelude::*; -use pyo3::PyVisit; - -#[pyclass] -struct TraverseTriesToTakePyRef {} - -#[pymethods] -impl TraverseTriesToTakePyRef { - fn __traverse__(slf: PyRef, visit: PyVisit) {} -} - -fn main() {} diff --git a/tests/ui/traverse_bare_self.stderr b/tests/ui/traverse_bare_self.stderr deleted file mode 100644 index aba76145dc3..00000000000 --- a/tests/ui/traverse_bare_self.stderr +++ /dev/null @@ -1,17 +0,0 @@ -error[E0308]: mismatched types - --> tests/ui/traverse_bare_self.rs:8:6 - | -7 | #[pymethods] - | ------------ arguments to this function are incorrect -8 | impl TraverseTriesToTakePyRef { - | ______^ -9 | | fn __traverse__(slf: PyRef, visit: PyVisit) {} - | |___________________^ expected fn pointer, found fn item - | - = note: expected fn pointer `for<'a, 'b> fn(&'a TraverseTriesToTakePyRef, PyVisit<'b>) -> Result<(), PyTraverseError>` - found fn item `for<'a, 'b> fn(pyo3::PyRef<'a, TraverseTriesToTakePyRef>, PyVisit<'b>) {TraverseTriesToTakePyRef::__traverse__}` -note: function defined here - --> src/impl_/pymethods.rs - | - | pub unsafe fn call_traverse_impl( - | ^^^^^^^^^^^^^^^^^^ From 7f0a98372a7d65b1453e06cea9cae8df642b260f Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Wed, 21 Jun 2023 08:00:26 +0200 Subject: [PATCH 40/59] Add Python::with_pool as a safer alternative to Python::new_pool. --- newsfragments/3263.added.md | 1 + src/marker.rs | 131 ++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 newsfragments/3263.added.md diff --git a/newsfragments/3263.added.md b/newsfragments/3263.added.md new file mode 100644 index 00000000000..888395d61f6 --- /dev/null +++ b/newsfragments/3263.added.md @@ -0,0 +1 @@ +Add the `Python::with_pool` which is a safer but more limited alternative to `Python::new_pool`. diff --git a/src/marker.rs b/src/marker.rs index 2e6cbe599a5..c4bdbbd06ac 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -940,6 +940,137 @@ impl<'py> Python<'py> { } } +impl Python<'_> { + /// Creates a scope using a new pool for managing PyO3's owned references. + /// + /// This is a safe alterantive to [`new_pool`][Self::new_pool] as + /// it limits the closure to using the new GIL token at the cost of + /// being unable to capture existing GIL-bound references. + /// + /// # Examples + /// + /// ```rust + /// # use pyo3::prelude::*; + /// Python::with_gil(|py| { + /// // Some long-running process like a webserver, which never releases the GIL. + /// loop { + /// // Create a new scope, so that PyO3 can clear memory at the end of the loop. + /// py.with_pool(|py| { + /// // do stuff... + /// }); + /// # break; // Exit the loop so that doctest terminates! + /// } + /// }); + /// ``` + /// + /// The `Ungil` bound on the closure does prevent hanging on to existing GIL-bound references + /// + /// ```compile_fail + /// # use pyo3::prelude::*; + /// # use pyo3::types::PyString; + /// + /// Python::with_gil(|py| { + /// let old_str = PyString::new(py, "a message from the past"); + /// + /// py.with_pool(|_py| { + /// print!("{:?}", old_str); + /// }); + /// }); + /// ``` + /// + /// or continuing to use the old GIL token + /// + /// ```compile_fail + /// # use pyo3::prelude::*; + /// + /// Python::with_gil(|old_py| { + /// old_py.with_pool(|_new_py| { + /// let _none = old_py.None(); + /// }); + /// }); + /// ``` + #[cfg(feature = "nightly")] + #[inline] + pub fn with_pool(&self, f: F) -> R + where + F: for<'py> FnOnce(Python<'py>) -> R + Ungil, + { + // SAFETY: The closure is `Ungil`, + // i.e. it does not capture any GIL-bound references + // and accesses only the newly created GIL token. + let pool = unsafe { GILPool::new() }; + + f(pool.python()) + } + + /// Creates a scope using a new pool for managing PyO3's owned references. + /// + /// This is a safer alterantive to [`new_pool`][Self::new_pool] as + /// it limits the closure to using the new GIL token at the cost of + /// being unable to capture existing GIL-bound references. + /// + /// # Examples + /// + /// ```rust + /// # use pyo3::prelude::*; + /// Python::with_gil(|py| { + /// // Some long-running process like a webserver, which never releases the GIL. + /// loop { + /// // Create a new scope, so that PyO3 can clear memory at the end of the loop. + /// unsafe { + /// py.with_pool(|py| { + /// // do stuff... + /// }); + /// } + /// # break; // Exit the loop so that doctest terminates! + /// } + /// }); + /// ``` + /// + /// The `Ungil` bound on the closure does prevent hanging on to existing GIL-bound references + /// + /// ```compile_fail + /// # use pyo3::prelude::*; + /// # use pyo3::types::PyString; + /// + /// Python::with_gil(|py| { + /// let old_str = PyString::new(py, "a message from the past"); + /// + /// py.with_pool(|_py| { + /// print!("{:?}", old_str); + /// }); + /// }); + /// ``` + /// + /// or continuing to use the old GIL token + /// + /// ```compile_fail + /// # use pyo3::prelude::*; + /// + /// Python::with_gil(|old_py| { + /// old_py.with_pool(|_new_py| { + /// let _none = old_py.None(); + /// }); + /// }); + /// ``` + /// + /// # Safety + /// + /// However, due to the `SendWrapper` loophole describe in the documentation of the [`Ungil`] trait, + /// this is still an unsafe API. Usage that does not involve runtime checking of thread affinity + /// should be practically safe. + #[cfg(not(feature = "nightly"))] + #[inline] + pub unsafe fn with_pool(&self, f: F) -> R + where + F: for<'py> FnOnce(Python<'py>) -> R + Ungil, + { + let pool = GILPool::new(); + + f(pool.python()) + } +} + impl<'unbound> Python<'unbound> { /// Unsafely creates a Python token with an unbounded lifetime. /// From ae27b3ba5e011effbae7d9d73eed076302218ca2 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Wed, 21 Jun 2023 09:12:10 +0200 Subject: [PATCH 41/59] Mark Python::with_pool as safe on stable. --- src/marker.rs | 71 +++------------------------------------------------ 1 file changed, 3 insertions(+), 68 deletions(-) diff --git a/src/marker.rs b/src/marker.rs index c4bdbbd06ac..e5b3ed81ca5 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -947,6 +947,9 @@ impl Python<'_> { /// it limits the closure to using the new GIL token at the cost of /// being unable to capture existing GIL-bound references. /// + /// Note that on stable Rust, this API suffers from the same the `SendWrapper` loophole + /// as [`allow_threads`][Self::allow_threads], c.f. the documentation of the [`Ungil`] trait, + /// /// # Examples /// /// ```rust @@ -989,7 +992,6 @@ impl Python<'_> { /// }); /// }); /// ``` - #[cfg(feature = "nightly")] #[inline] pub fn with_pool(&self, f: F) -> R where @@ -1002,73 +1004,6 @@ impl Python<'_> { f(pool.python()) } - - /// Creates a scope using a new pool for managing PyO3's owned references. - /// - /// This is a safer alterantive to [`new_pool`][Self::new_pool] as - /// it limits the closure to using the new GIL token at the cost of - /// being unable to capture existing GIL-bound references. - /// - /// # Examples - /// - /// ```rust - /// # use pyo3::prelude::*; - /// Python::with_gil(|py| { - /// // Some long-running process like a webserver, which never releases the GIL. - /// loop { - /// // Create a new scope, so that PyO3 can clear memory at the end of the loop. - /// unsafe { - /// py.with_pool(|py| { - /// // do stuff... - /// }); - /// } - /// # break; // Exit the loop so that doctest terminates! - /// } - /// }); - /// ``` - /// - /// The `Ungil` bound on the closure does prevent hanging on to existing GIL-bound references - /// - /// ```compile_fail - /// # use pyo3::prelude::*; - /// # use pyo3::types::PyString; - /// - /// Python::with_gil(|py| { - /// let old_str = PyString::new(py, "a message from the past"); - /// - /// py.with_pool(|_py| { - /// print!("{:?}", old_str); - /// }); - /// }); - /// ``` - /// - /// or continuing to use the old GIL token - /// - /// ```compile_fail - /// # use pyo3::prelude::*; - /// - /// Python::with_gil(|old_py| { - /// old_py.with_pool(|_new_py| { - /// let _none = old_py.None(); - /// }); - /// }); - /// ``` - /// - /// # Safety - /// - /// However, due to the `SendWrapper` loophole describe in the documentation of the [`Ungil`] trait, - /// this is still an unsafe API. Usage that does not involve runtime checking of thread affinity - /// should be practically safe. - #[cfg(not(feature = "nightly"))] - #[inline] - pub unsafe fn with_pool(&self, f: F) -> R - where - F: for<'py> FnOnce(Python<'py>) -> R + Ungil, - { - let pool = GILPool::new(); - - f(pool.python()) - } } impl<'unbound> Python<'unbound> { From 839c9a04c55dfcec4d4d0cf4816e2d1b0813dc66 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 21 Jun 2023 22:43:38 +0100 Subject: [PATCH 42/59] get all badges through shields.io --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d6e3e75d728..5b25101bf20 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # PyO3 -[![actions status](https://github.com/PyO3/pyo3/workflows/CI/badge.svg)](https://github.com/PyO3/pyo3/actions) -[![benchmark](https://github.com/PyO3/pyo3/actions/workflows/bench.yml/badge.svg)](https://pyo3.rs/dev/bench/) -[![codecov](https://codecov.io/gh/PyO3/pyo3/branch/main/graph/badge.svg)](https://codecov.io/gh/PyO3/pyo3) -[![crates.io](https://img.shields.io/crates/v/pyo3)](https://crates.io/crates/pyo3) +[![actions status](https://img.shields.io/github/actions/workflow/status/PyO3/pyo3/ci.yml?branch=main&logo=github&style=)](https://github.com/PyO3/pyo3/actions) +[![benchmark](https://img.shields.io/badge/benchmark-✓-Green?logo=github)](https://pyo3.rs/dev/bench/) +[![codecov](https://img.shields.io/codecov/c/gh/PyO3/pyo3?logo=codecov)](https://codecov.io/gh/PyO3/pyo3) +[![crates.io](https://img.shields.io/crates/v/pyo3?logo=rust)](https://crates.io/crates/pyo3) [![minimum rustc 1.48](https://img.shields.io/badge/rustc-1.48+-blue.svg)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) -[![dev chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/PyO3/Lobby) -[![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) +[![dev chat](https://img.shields.io/gitter/room/PyO3/Lobby?logo=gitter)](https://gitter.im/PyO3/Lobby) +[![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green?logo=github)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) [Rust](https://www.rust-lang.org/) bindings for [Python](https://www.python.org/), including tools for creating native Python extension modules. Running and interacting with Python code from a Rust binary is also supported. From 511003b13fc23f1f4e23a830673a01baf76f11f0 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 21 Jun 2023 21:35:06 +0100 Subject: [PATCH 43/59] add http headers to help search engines index the docs --- .netlify/build.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.netlify/build.sh b/.netlify/build.sh index 215ee8c8c6d..6cab914b78c 100755 --- a/.netlify/build.sh +++ b/.netlify/build.sh @@ -17,21 +17,45 @@ mv pyo3-gh-pages netlify_build ## Configure netlify _redirects file # Add redirect for each documented version +set +x # these loops get very spammy and fill the deploy log + for d in netlify_build/v*; do version="${d/netlify_build\/v/}" echo "/v$version/doc/* https://docs.rs/pyo3/$version/:splat" >> netlify_build/_redirects + if [ $version != $PYO3_VERSION ]; then + # for old versions, mark the files in the latest version as the canonical URL + for file in $(find $d -type f); do + file_path="${file/$d\//}" + # remove index.html and/or .html suffix to match the page URL on the + # final netlfiy site + url_path="$file_path" + if [[ $file_path == index.html ]]; then + url_path="" + elif [[ $file_path == *.html ]]; then + url_path="${file_path%.html}" + fi + echo "/v$version/$url_path" >> netlify_build/_headers + if test -f "netlify_build/v$PYO3_VERSION/$file_path"; then + echo " Link: ; rel=\"canonical\"" >> netlify_build/_headers + else + # this file doesn't exist in the latest guide, don't index it + echo " X-Robots-Tag: noindex" >> netlify_build/_headers + fi + done + fi done # Add latest redirect -echo "/latest/* /v${PYO3_VERSION}/:splat" >> netlify_build/_redirects +echo "/latest/* /v${PYO3_VERSION}/:splat 302" >> netlify_build/_redirects ## Add landing page redirect if [ "${CONTEXT}" == "deploy-preview" ]; then echo "/ /main/" >> netlify_build/_redirects else - echo "/ /v${PYO3_VERSION}/" >> netlify_build/_redirects + echo "/ /v${PYO3_VERSION}/ 302" >> netlify_build/_redirects fi +set -x ## Generate towncrier release notes pip install towncrier From 5da4305f99204eb683ee3da2254d1d86b20b9bec Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 25 Jun 2023 19:26:08 +0100 Subject: [PATCH 44/59] add `PyDict::get_item_with_error` for PyPy --- newsfragments/3270.added.md | 1 + pyo3-ffi/src/dictobject.rs | 1 + src/types/dict.rs | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3270.added.md diff --git a/newsfragments/3270.added.md b/newsfragments/3270.added.md new file mode 100644 index 00000000000..66b2db9d83d --- /dev/null +++ b/newsfragments/3270.added.md @@ -0,0 +1 @@ +Add `PyDict::get_item_with_error` on PyPy. diff --git a/pyo3-ffi/src/dictobject.rs b/pyo3-ffi/src/dictobject.rs index b03fbb303a8..aa9435f6170 100644 --- a/pyo3-ffi/src/dictobject.rs +++ b/pyo3-ffi/src/dictobject.rs @@ -23,6 +23,7 @@ extern "C" { pub fn PyDict_New() -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyDict_GetItem")] pub fn PyDict_GetItem(mp: *mut PyObject, key: *mut PyObject) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyDict_GetItemWithError")] pub fn PyDict_GetItemWithError(mp: *mut PyObject, key: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyDict_SetItem")] pub fn PyDict_SetItem(mp: *mut PyObject, key: *mut PyObject, item: *mut PyObject) -> c_int; diff --git a/src/types/dict.rs b/src/types/dict.rs index 0aedb13a76b..1decebcc60e 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -157,7 +157,6 @@ impl PyDict { /// returns `Ok(None)` if item is not present, or `Err(PyErr)` if an error occurs. /// /// To get a `KeyError` for non-existing keys, use `PyAny::get_item_with_error`. - #[cfg(not(PyPy))] pub fn get_item_with_error(&self, key: K) -> PyResult> where K: ToPyObject, From fbfa09fd68ed626c62eae867831760c52a8ceed9 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 25 Jun 2023 19:26:51 +0100 Subject: [PATCH 45/59] refactor `PyDict::get_item[_with_error]` implementations --- src/types/dict.rs | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/src/types/dict.rs b/src/types/dict.rs index 1decebcc60e..b4ed14c3f89 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -2,10 +2,9 @@ use super::PyMapping; use crate::err::{self, PyErr, PyResult}; use crate::ffi::Py_ssize_t; use crate::types::{PyAny, PyList}; -use crate::{ffi, AsPyPointer, Python, ToPyObject}; #[cfg(not(PyPy))] -use crate::{IntoPyPointer, PyObject}; -use std::ptr::NonNull; +use crate::IntoPyPointer; +use crate::{ffi, AsPyPointer, PyObject, Python, ToPyObject}; /// Represents a Python `dict`. #[repr(transparent)] @@ -143,12 +142,16 @@ impl PyDict { where K: ToPyObject, { + self.get_item_impl(key.to_object(self.py())) + } + + fn get_item_impl(&self, key: PyObject) -> Option<&PyAny> { + let py = self.py(); unsafe { - let ptr = ffi::PyDict_GetItem(self.as_ptr(), key.to_object(self.py()).as_ptr()); - NonNull::new(ptr).map(|p| { - // PyDict_GetItem return s borrowed ptr, must make it owned for safety (see #890). - self.py().from_owned_ptr(ffi::_Py_NewRef(p.as_ptr())) - }) + let ptr = ffi::PyDict_GetItem(self.as_ptr(), key.as_ptr()); + // PyDict_GetItem returns a borrowed ptr, must make it owned for safety (see #890). + // PyObject::from_borrowed_ptr_or_opt will take ownership in this way. + PyObject::from_borrowed_ptr_or_opt(py, ptr).map(|pyobject| pyobject.into_ref(py)) } } @@ -161,14 +164,19 @@ impl PyDict { where K: ToPyObject, { - unsafe { - let ptr = - ffi::PyDict_GetItemWithError(self.as_ptr(), key.to_object(self.py()).as_ptr()); - if !ffi::PyErr_Occurred().is_null() { - return Err(PyErr::fetch(self.py())); - } + self.get_item_with_error_impl(key.to_object(self.py())) + } - Ok(NonNull::new(ptr).map(|p| self.py().from_owned_ptr(ffi::_Py_NewRef(p.as_ptr())))) + fn get_item_with_error_impl(&self, key: PyObject) -> PyResult> { + let py = self.py(); + unsafe { + let ptr = ffi::PyDict_GetItemWithError(self.as_ptr(), key.as_ptr()); + // PyDict_GetItemWithError returns a borrowed ptr, must make it owned for safety (see #890). + // PyObject::from_borrowed_ptr_or_opt will take ownership in this way. + PyObject::from_borrowed_ptr_or_opt(py, ptr) + .map(|pyobject| Ok(pyobject.into_ref(py))) + .or_else(|| PyErr::take(py).map(Err)) + .transpose() } } From 60e55bb62cfce63bf1bed952a96509248e85cfd8 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sun, 25 Jun 2023 21:11:00 +0200 Subject: [PATCH 46/59] Our PR template still referenced the removed xtask infrastructure. --- .github/pull_request_template.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f9877a1ca40..02c0d5d5c90 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,5 +10,5 @@ Please consider adding the following to your pull request: - tests for all new or changed functions PyO3's CI pipeline will check your pull request. To run its tests -locally, you can run ```cargo xtask ci```. See its documentation - [here](https://github.com/PyO3/pyo3/tree/main/xtask#readme). +locally, you can run ```nox```. See ```nox --list-sessions``` +for a list of supported actions. From c66f289c0e178b351ca70fbc4474b42641d0da10 Mon Sep 17 00:00:00 2001 From: Luca Trevisani Date: Sun, 25 Jun 2023 20:40:51 +0200 Subject: [PATCH 47/59] Add conversions between Rust Path/PathBuf and Python str / pathlib.Path --- guide/src/conversions/tables.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index 38ed27c1f1e..911a9869973 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -13,7 +13,7 @@ The table below contains the Python type and the corresponding function argument | Python | Rust | Rust (Python-native) | | ------------- |:-------------------------------:|:--------------------:| | `object` | - | `&PyAny` | -| `str` | `String`, `Cow`, `&str`, `OsString`, `PathBuf` | `&PyUnicode` | +| `str` | `String`, `Cow`, `&str`, `OsString`, `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | | `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `&PyBytes` | | `bool` | `bool` | `&PyBool` | | `int` | Any integer type (`i32`, `u32`, `usize`, etc) | `&PyLong` | @@ -28,6 +28,7 @@ The table below contains the Python type and the corresponding function argument | `slice` | - | `&PySlice` | | `type` | - | `&PyType` | | `module` | - | `&PyModule` | +| `pathlib.Path` | `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | | `datetime.datetime` | - | `&PyDateTime` | | `datetime.date` | - | `&PyDate` | | `datetime.time` | - | `&PyTime` | From 1cc563e6de18e8146de4a932a38e07bfb496e2a3 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Mon, 26 Jun 2023 23:14:32 +0200 Subject: [PATCH 48/59] Extended range of supported versions of `indexmap` optional dependency to include version 2. --- Cargo.toml | 2 +- newsfragments/3277.packaging.md | 1 + noxfile.py | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 newsfragments/3277.packaging.md diff --git a/Cargo.toml b/Cargo.toml index 59b01d38000..599beda3659 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ anyhow = { version = "1.0", optional = true } chrono = { version = "0.4", default-features = false, optional = true } eyre = { version = ">= 0.4, < 0.7", optional = true } hashbrown = { version = ">= 0.9, < 0.15", optional = true } -indexmap = { version = "1.6", optional = true } +indexmap = { version = ">= 1.6, < 3", optional = true } num-bigint = { version = "0.4", optional = true } num-complex = { version = ">= 0.2, < 0.5", optional = true } rust_decimal = { version = "1.0.0", default-features = false, optional = true } diff --git a/newsfragments/3277.packaging.md b/newsfragments/3277.packaging.md new file mode 100644 index 00000000000..1fbed18467d --- /dev/null +++ b/newsfragments/3277.packaging.md @@ -0,0 +1 @@ +Extended range of supported versions of `indexmap` optional dependency to include version 2. diff --git a/noxfile.py b/noxfile.py index ec3f62d8d8c..b6607da100e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -34,7 +34,7 @@ def test_rust(session: nox.Session): _run_cargo_test(session) _run_cargo_test(session, features="abi3") - if not "skip-full" in session.posargs: + if "skip-full" not in session.posargs: _run_cargo_test(session, features="full") _run_cargo_test(session, features="abi3 full") @@ -338,7 +338,7 @@ def format_guide(session: nox.Session): for path in Path("guide").glob("**/*.md"): session.log("Working on %s", path) - content = path.read_text() + path.read_text() lines = iter(path.read_text().splitlines(True)) new_lines = [] @@ -469,6 +469,8 @@ def set_minimal_package_versions(session: nox.Session): "rayon": "1.6.1", "rayon-core": "1.10.2", "regex": "1.7.3", + "proptest": "1.0.0", + "indexmap": "1.9.3", } # run cargo update first to ensure that everything is at highest From 32587ec8b23301337de282d53ca305cdc88d80fb Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 25 Jun 2023 19:12:14 +0100 Subject: [PATCH 49/59] stop suppressing unrelated exceptions in `PyAny::hasattr` --- newsfragments/3271.fixed.md | 1 + src/types/any.rs | 85 +++++++++++++++++++++++++++++++++---- 2 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 newsfragments/3271.fixed.md diff --git a/newsfragments/3271.fixed.md b/newsfragments/3271.fixed.md new file mode 100644 index 00000000000..ff3fe71517f --- /dev/null +++ b/newsfragments/3271.fixed.md @@ -0,0 +1 @@ +Stop suppressing unrelated exceptions in `PyAny::hasattr`. diff --git a/src/types/any.rs b/src/types/any.rs index a545d9b2ac1..f448e43bf05 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1,7 +1,7 @@ use crate::class::basic::CompareOp; use crate::conversion::{AsPyPointer, FromPyObject, IntoPy, IntoPyPointer, PyTryFrom, ToPyObject}; use crate::err::{PyDowncastError, PyErr, PyResult}; -use crate::exceptions::PyTypeError; +use crate::exceptions::{PyAttributeError, PyTypeError}; use crate::type_object::PyTypeInfo; #[cfg(not(PyPy))] use crate::types::PySuper; @@ -79,14 +79,37 @@ impl PyAny { /// /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `attr_name`. + /// + /// # Example: `intern!`ing the attribute name + /// + /// ``` + /// # use pyo3::{intern, pyfunction, types::PyModule, Python, PyResult}; + /// # + /// #[pyfunction] + /// fn has_version(sys: &PyModule) -> PyResult { + /// sys.hasattr(intern!(sys.py(), "version")) + /// } + /// # + /// # Python::with_gil(|py| { + /// # let sys = py.import("sys").unwrap(); + /// # has_version(sys).unwrap(); + /// # }); + /// ``` pub fn hasattr(&self, attr_name: N) -> PyResult where N: IntoPy>, { - let py = self.py(); - let attr_name = attr_name.into_py(py); + fn inner(any: &PyAny, attr_name: Py) -> PyResult { + // PyObject_HasAttr suppresses all exceptions, which was the behaviour of `hasattr` in Python 2. + // Use an implementation which suppresses only AttributeError, which is consistent with `hasattr` in Python 3. + match any._getattr(attr_name) { + Ok(_) => Ok(true), + Err(err) if err.is_instance_of::(any.py()) => Ok(false), + Err(e) => Err(e), + } + } - unsafe { Ok(ffi::PyObject_HasAttr(self.as_ptr(), attr_name.as_ptr()) != 0) } + inner(self, attr_name.into_py(self.py())) } /// Retrieves an attribute value. @@ -115,12 +138,20 @@ impl PyAny { where N: IntoPy>, { - let py = self.py(); - let attr_name = attr_name.into_py(py); + fn inner(any: &PyAny, attr_name: Py) -> PyResult<&PyAny> { + any._getattr(attr_name) + .map(|object| object.into_ref(any.py())) + } + + inner(self, attr_name.into_py(self.py())) + } + fn _getattr(&self, attr_name: Py) -> PyResult { unsafe { - let ret = ffi::PyObject_GetAttr(self.as_ptr(), attr_name.as_ptr()); - py.from_owned_ptr_or_err(ret) + Py::from_owned_ptr_or_err( + self.py(), + ffi::PyObject_GetAttr(self.as_ptr(), attr_name.as_ptr()), + ) } } @@ -1169,6 +1200,44 @@ class SimpleClass: }); } + #[test] + fn test_hasattr() { + Python::with_gil(|py| { + let x = 5.to_object(py).into_ref(py); + assert!(x.is_instance_of::()); + + assert!(x.hasattr("to_bytes").unwrap()); + assert!(!x.hasattr("bbbbbbytes").unwrap()); + }) + } + + #[cfg(feature = "macros")] + #[test] + fn test_hasattr_error() { + use crate::exceptions::PyValueError; + use crate::prelude::*; + + #[pyclass(crate = "crate")] + struct GetattrFail; + + #[pymethods(crate = "crate")] + impl GetattrFail { + fn __getattr__(&self, attr: PyObject) -> PyResult { + Err(PyValueError::new_err(attr)) + } + } + + Python::with_gil(|py| { + let obj = Py::new(py, GetattrFail).unwrap(); + let obj = obj.as_ref(py).as_ref(); + + assert!(obj + .hasattr("foo") + .unwrap_err() + .is_instance_of::(py)); + }) + } + #[test] fn test_nan_eq() { Python::with_gil(|py| { From cd33b1cd98b36ff4f1b21a914f5cd88258d072c9 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 26 Jun 2023 22:26:37 +0100 Subject: [PATCH 50/59] move pyo3-ffi-check into its own workspace --- .github/workflows/build.yml | 2 +- Cargo.toml | 2 -- noxfile.py | 23 ++++++++++++----------- pyo3-ffi-check/Cargo.toml | 7 ++++++- pyo3-ffi-check/macro/Cargo.toml | 2 +- pyo3-ffi-check/src/main.rs | 3 ++- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a003feffc13..5bf947fca13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -136,10 +136,10 @@ jobs: - '.github/workflows/build.yml' - name: Run pyo3-ffi-check - run: nox -s ffi-check # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor # is pypy 3.9 on windows if: ${{ steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows')) }} + run: nox -s ffi-check - name: Test cross compilation diff --git a/Cargo.toml b/Cargo.toml index 599beda3659..115375af7b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,8 +176,6 @@ harness = false [workspace] members = [ "pyo3-ffi", - "pyo3-ffi-check", - "pyo3-ffi-check/macro", "pyo3-build-config", "pyo3-macros", "pyo3-macros-backend", diff --git a/noxfile.py b/noxfile.py index b6607da100e..f0c52ac60fd 100644 --- a/noxfile.py +++ b/noxfile.py @@ -76,7 +76,8 @@ def fmt(session: nox.Session): @nox.session(name="fmt-rust", venv_backend="none") def fmt_rust(session: nox.Session): - _run(session, "cargo", "fmt", "--all", "--check", external=True) + _run_cargo(session, "fmt", "--all", "--check") + _run_cargo(session, "fmt", *_FFI_CHECK, "--all", "--check") @nox.session(name="fmt-py") @@ -103,9 +104,6 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: *feature_set, "--all-targets", "--workspace", - # linting pyo3-ffi-check requires docs to have been built or - # the macros will error; doesn't seem worth it on CI - "--exclude=pyo3-ffi-check", "--", "--deny=warnings", external=True, @@ -146,9 +144,6 @@ def _check(env: Dict[str, str]) -> None: *feature_set, "--all-targets", "--workspace", - # linting pyo3-ffi-check requires docs to have been built or - # the macros will error; doesn't seem worth it on CI - "--exclude=pyo3-ffi-check", external=True, env=env, ) @@ -338,8 +333,6 @@ def format_guide(session: nox.Session): for path in Path("guide").glob("**/*.md"): session.log("Working on %s", path) - path.read_text() - lines = iter(path.read_text().splitlines(True)) new_lines = [] @@ -533,8 +526,9 @@ def load_pkg_versions(): @nox.session(name="ffi-check") def ffi_check(session: nox.Session): - session.run("cargo", "doc", "-p", "pyo3-ffi", "--no-deps", external=True) - _run(session, "cargo", "run", "-p", "pyo3-ffi-check", external=True) + _run_cargo(session, "doc", *_FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") + _run_cargo(session, "clippy", "--workspace", "--all-targets", *_FFI_CHECK) + _run_cargo(session, "run", *_FFI_CHECK) @lru_cache() @@ -616,6 +610,10 @@ def _run(session: nox.Session, *args: str, **kwargs: Any) -> None: print("::endgroup::", file=sys.stderr) +def _run_cargo(session: nox.Session, *args: str, **kwargs: Any) -> None: + _run(session, "cargo", *args, **kwargs, external=True) + + def _run_cargo_test( session: nox.Session, *, @@ -685,3 +683,6 @@ def _job_with_config(implementation, version) -> bool: for version in PYPY_VERSIONS: _job_with_config("PyPy", version) + + +_FFI_CHECK = ("--manifest-path", "pyo3-ffi-check/Cargo.toml") diff --git a/pyo3-ffi-check/Cargo.toml b/pyo3-ffi-check/Cargo.toml index 9957bef8bf2..06829298ea3 100644 --- a/pyo3-ffi-check/Cargo.toml +++ b/pyo3-ffi-check/Cargo.toml @@ -13,5 +13,10 @@ path = "../pyo3-ffi" features = ["extension-module"] # A lazy way of skipping linking in most cases (as we don't use any runtime symbols) [build-dependencies] -bindgen = "0.65.1" +bindgen = "0.66.1" pyo3-build-config = { path = "../pyo3-build-config" } + +[workspace] +members = [ + "macro" +] diff --git a/pyo3-ffi-check/macro/Cargo.toml b/pyo3-ffi-check/macro/Cargo.toml index a9f51f654c6..999342fa9c9 100644 --- a/pyo3-ffi-check/macro/Cargo.toml +++ b/pyo3-ffi-check/macro/Cargo.toml @@ -11,4 +11,4 @@ proc-macro = true glob = "0.3" quote = "1" proc-macro2 = "1" -scraper = "0.12" +scraper = "0.17" diff --git a/pyo3-ffi-check/src/main.rs b/pyo3-ffi-check/src/main.rs index 36af5c160a4..91a7dca6ee5 100644 --- a/pyo3-ffi-check/src/main.rs +++ b/pyo3-ffi-check/src/main.rs @@ -78,7 +78,8 @@ fn main() { non_camel_case_types, non_upper_case_globals, dead_code, - improper_ctypes + improper_ctypes, + clippy::all )] mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); From 0ba07db807559c9d1dfb8919e58ba9f21d83da05 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 27 Jun 2023 22:57:01 +0100 Subject: [PATCH 51/59] use _run_cargo helper in noxfile --- noxfile.py | 41 +++++++++++------------------------------ 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/noxfile.py b/noxfile.py index f0c52ac60fd..f332b98f30b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -49,11 +49,10 @@ def test_py(session: nox.Session) -> None: @nox.session(venv_backend="none") def coverage(session: nox.Session) -> None: session.env.update(_get_coverage_env()) - _run(session, "cargo", "llvm-cov", "clean", "--workspace", external=True) + _run_cargo(session, "llvm-cov", "clean", "--workspace") test(session) - _run( + _run_cargo( session, - "cargo", "llvm-cov", "--package=pyo3", "--package=pyo3-build-config", @@ -64,7 +63,6 @@ def coverage(session: nox.Session) -> None: "--codecov", "--output-path", "coverage.json", - external=True, ) @@ -97,16 +95,14 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: env = env or os.environ for feature_set in _get_feature_sets(): try: - _run( + _run_cargo( session, - "cargo", "clippy", *feature_set, "--all-targets", "--workspace", "--", "--deny=warnings", - external=True, env=env, ) except Exception: @@ -134,17 +130,14 @@ def check_all(session: nox.Session) -> None: def _check(env: Dict[str, str]) -> None: nonlocal success - env = env or os.environ for feature_set in _get_feature_sets(): try: - _run( + _run_cargo( session, - "cargo", "check", *feature_set, "--all-targets", "--workspace", - external=True, env=env, ) except Exception: @@ -307,9 +300,8 @@ def docs(session: nox.Session) -> None: rustdoc_flags.append(session.env.get("RUSTDOCFLAGS", "")) session.env["RUSTDOCFLAGS"] = " ".join(rustdoc_flags) - _run( + _run_cargo( session, - "cargo", *toolchain_flags, "doc", "--lib", @@ -318,7 +310,6 @@ def docs(session: nox.Session) -> None: "--no-deps", "--workspace", *cargo_flags, - external=True, ) @@ -379,9 +370,8 @@ def format_guide(session: nox.Session): @nox.session(name="address-sanitizer", venv_backend="none") def address_sanitizer(session: nox.Session): - _run( + _run_cargo( session, - "cargo", "+nightly", "test", "--release", @@ -394,7 +384,6 @@ def address_sanitizer(session: nox.Session): "RUSTDOCFLAGS": "-Zsanitizer=address", "ASAN_OPTIONS": "detect_leaks=0", }, - external=True, ) @@ -470,15 +459,9 @@ def set_minimal_package_versions(session: nox.Session): # possible version, so that this matches what CI will resolve to. for project in projects: if project is None: - _run(session, "cargo", "update", external=True) + _run_cargo(session, "update") else: - _run( - session, - "cargo", - "update", - f"--manifest-path={project}/Cargo.toml", - external=True, - ) + _run_cargo(session, "update", f"--manifest-path={project}/Cargo.toml") for project in projects: lock_file = Path(project or "") / "Cargo.lock" @@ -512,15 +495,13 @@ def load_pkg_versions(): # supported on MSRV for project in projects: if project is None: - _run(session, "cargo", "metadata", silent=True, external=True) + _run_cargo(session, "metadata", silent=True) else: - _run( + _run_cargo( session, - "cargo", "metadata", f"--manifest-path={project}/Cargo.toml", silent=True, - external=True, ) @@ -635,7 +616,7 @@ def _run_cargo_test( def _run_cargo_publish(session: nox.Session, *, package: str) -> None: - _run(session, "cargo", "publish", f"--package={package}", external=True) + _run_cargo(session, "publish", f"--package={package}") def _run_cargo_set_package_version( From 710a2dc1da1878aff8b24edb1b7680333618bc9b Mon Sep 17 00:00:00 2001 From: CallMeMSL <36897759+CallMeMSL@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:46:13 +0200 Subject: [PATCH 52/59] Fix typos and add punctuation recommended by grammarly in Class.md --- guide/src/class.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index 4b1b7c55165..15dbe0cbdf1 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -60,7 +60,7 @@ To integrate Rust types with Python, PyO3 needs to place some restrictions on th Rust lifetimes are used by the Rust compiler to reason about a program's memory safety. They are a compile-time only concept; there is no way to access Rust lifetimes at runtime from a dynamic language like Python. -As soon as Rust data is exposed to Python, there is no guarantee which the Rust compiler can make on how long the data will live. Python is a reference-counted language and those references can be held for an arbitrarily long time which is untraceable by the Rust compiler. The only possible way to express this correctly is to require that any `#[pyclass]` does not borrow data for any lifetime shorter than the `'static` lifetime, i.e. the `#[pyclass]` cannot have any lifetime parameters. +As soon as Rust data is exposed to Python, there is no guarantee that the Rust compiler can make on how long the data will live. Python is a reference-counted language and those references can be held for an arbitrarily long time which is untraceable by the Rust compiler. The only possible way to express this correctly is to require that any `#[pyclass]` does not borrow data for any lifetime shorter than the `'static` lifetime, i.e. the `#[pyclass]` cannot have any lifetime parameters. When you need to share ownership of data between Python and Rust, instead of using borrowed references with lifetimes consider using reference-counted smart pointers such as [`Arc`] or [`Py`]. @@ -74,7 +74,7 @@ Because Python objects are freely shared between threads by the Python interpret ## Constructor -By default it is not possible to create an instance of a custom class from Python code. +By default, it is not possible to create an instance of a custom class from Python code. To declare a constructor, you need to define a method and annotate it with the `#[new]` attribute. Only Python's `__new__` method can be specified, `__init__` is not available. @@ -114,7 +114,7 @@ impl Nonzero { ``` As you can see, the Rust method name is not important here; this way you can -still use `new()` for a Rust-level constructor. +still, use `new()` for a Rust-level constructor. If no method marked with `#[new]` is declared, object instances can only be created from Rust, but not from Python. @@ -266,7 +266,7 @@ use the `extends` parameter for `pyclass` with the full path to the base class. For convenience, `(T, U)` implements `Into>` where `U` is the base class of `T`. -But for more deeply nested inheritance, you have to return `PyClassInitializer` +But for a more deeply nested inheritance, you have to return `PyClassInitializer` explicitly. To get a parent class from a child, use [`PyRef`] instead of `&self` for methods, @@ -743,8 +743,8 @@ fn print_field(my_class: PyRef<'_, MyClass>) { println!("{}", my_class.my_field); } -// Take a GIL-bound reference to the underyling cell -// when borrowing needs to be managed manaually. +// Take a GIL-bound reference to the underlying cell +// when borrowing needs to be managed manually. #[pyfunction] fn increment_then_print_field(my_class: &PyCell) { my_class.borrow_mut().my_field += 1; @@ -759,7 +759,7 @@ fn print_refcnt(my_class: Py, py: Python<'_>) { } ``` -Classes can also be passed by value if they can be be cloned, i.e. they automatically implement `FromPyObject` if they implement `Clone`, e.g. via `#[derive(Clone)]`: +Classes can also be passed by value if they can be cloned, i.e. they automatically implement `FromPyObject` if they implement `Clone`, e.g. via `#[derive(Clone)]`: ```rust # #![allow(dead_code)] @@ -783,7 +783,7 @@ Note that `#[derive(FromPyObject)]` on a class is usually not useful as it tries Similar to `#[pyfunction]`, the `#[pyo3(signature = (...))]` attribute can be used to specify the way that `#[pymethods]` accept arguments. Consult the documentation for [`function signatures`](./function/signature.md) to see the parameters this attribute accepts. -The following example defines a class `MyClass` with a method `method`. This method has a signature which sets default values for `num` and `name`, and indicates that `py_args` should collect all extra positional arguments and `py_kwargs` all extra keyword arguments: +The following example defines a class `MyClass` with a method `method`. This method has a signature that sets default values for `num` and `name`, and indicates that `py_args` should collect all extra positional arguments and `py_kwargs` all extra keyword arguments: ```rust # use pyo3::prelude::*; @@ -819,7 +819,7 @@ impl MyClass { } ``` -In Python this might be used like: +In Python, this might be used like: ```python >>> import mymodule From efea50bbfc7fd56ad1e51f2fdbbb7fc92793eccf Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 28 Jun 2023 08:38:12 +0100 Subject: [PATCH 53/59] clarify ownership of opt_to_pyobj helper function --- src/types/datetime.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 2c995356e2c..8d5aae1f977 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -257,7 +257,7 @@ impl PyDateTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), api.DateTimeType, ); py.from_owned_ptr_or_err(ptr) @@ -294,7 +294,7 @@ impl PyDateTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), c_int::from(fold), api.DateTimeType, ); @@ -399,7 +399,7 @@ impl PyTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), api.TimeType, ); py.from_owned_ptr_or_err(ptr) @@ -423,7 +423,7 @@ impl PyTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), fold as c_int, api.TimeType, ); @@ -535,12 +535,12 @@ impl PyDeltaAccess for PyDelta { } } -// Utility function -fn opt_to_pyobj(py: Python<'_>, opt: Option<&PyTzInfo>) -> *mut ffi::PyObject { - // Convenience function for unpacking Options to either an Object or None +// Utility function which returns a borrowed reference to either +// the underlying tzinfo or None. +fn opt_to_pyobj(opt: Option<&PyTzInfo>) -> *mut ffi::PyObject { match opt { Some(tzi) => tzi.as_ptr(), - None => py.None().as_ptr(), + None => unsafe { ffi::Py_None() }, } } From 18122740bd7cdee435f06a11130e231e6f86bba9 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sun, 2 Jul 2023 20:05:18 +0200 Subject: [PATCH 54/59] Apparently, PySet_Add does not steal a reference, hence we should not forget to clean up ours. --- newsfragments/3286.fixed.md | 1 + src/types/frozenset.rs | 4 ++-- src/types/set.rs | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3286.fixed.md diff --git a/newsfragments/3286.fixed.md b/newsfragments/3286.fixed.md new file mode 100644 index 00000000000..cfe647b9eb0 --- /dev/null +++ b/newsfragments/3286.fixed.md @@ -0,0 +1 @@ +Fix memory leak when creating `PySet` or `PyFrozenSet` or returning types converted into these internally, e.g. `HashSet` or `BTreeSet`. diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index ecfbbb430c5..ff91be9251e 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -2,7 +2,7 @@ use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, - IntoPyPointer, Py, PyObject, + Py, PyObject, }; use crate::{ffi, AsPyPointer, PyAny, Python, ToPyObject}; @@ -209,7 +209,7 @@ pub(crate) fn new_from_iter( for obj in elements { unsafe { - err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.into_ptr()))?; + err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.as_ptr()))?; } } diff --git a/src/types/set.rs b/src/types/set.rs index cf043d99dae..af174e1b7c5 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -2,7 +2,7 @@ use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, - IntoPyPointer, Py, + Py, }; use crate::{ffi, AsPyPointer, PyAny, PyObject, Python, ToPyObject}; use std::ptr; @@ -250,7 +250,7 @@ pub(crate) fn new_from_iter( for obj in elements { unsafe { - err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.into_ptr()))?; + err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.as_ptr()))?; } } From 9f9be35102f80bae8337e38069d2516baa110378 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 2 Jul 2023 17:26:31 -0400 Subject: [PATCH 55/59] Allow `#[new]` to return existing instances fixes #2384 --- guide/src/class.md | 3 ++ newsfragments/3287.added.md | 1 + src/pyclass_init.rs | 31 ++++++++++++++----- tests/test_class_new.rs | 60 +++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 newsfragments/3287.added.md diff --git a/guide/src/class.md b/guide/src/class.md index 15dbe0cbdf1..b5a27b517cc 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -113,6 +113,9 @@ impl Nonzero { } ``` +If you want to return an existing object (for example, because your `new` +method caches the values it returns), `new` can return `pyo3::Py`. + As you can see, the Rust method name is not important here; this way you can still, use `new()` for a Rust-level constructor. diff --git a/newsfragments/3287.added.md b/newsfragments/3287.added.md new file mode 100644 index 00000000000..bde61a4b506 --- /dev/null +++ b/newsfragments/3287.added.md @@ -0,0 +1 @@ +`#[new]` methods may now return `Py` in order to return existing instances diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 015f5791579..0f606853f74 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -1,7 +1,7 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::callback::IntoPyCallbackOutput; use crate::impl_::pyclass::{PyClassBaseType, PyClassDict, PyClassThreadChecker, PyClassWeakRef}; -use crate::{ffi, PyCell, PyClass, PyErr, PyResult, Python}; +use crate::{ffi, IntoPyPointer, Py, PyCell, PyClass, PyErr, PyResult, Python}; use crate::{ ffi::PyTypeObject, pycell::{ @@ -137,9 +137,14 @@ impl PyObjectInit for PyNativeTypeInitializer { /// ); /// }); /// ``` -pub struct PyClassInitializer { - init: T, - super_init: ::Initializer, +pub struct PyClassInitializer(PyClassInitializerImpl); + +enum PyClassInitializerImpl { + Existing(Py), + New { + init: T, + super_init: ::Initializer, + }, } impl PyClassInitializer { @@ -147,7 +152,7 @@ impl PyClassInitializer { /// /// It is recommended to use `add_subclass` instead of this method for most usage. pub fn new(init: T, super_init: ::Initializer) -> Self { - Self { init, super_init } + Self(PyClassInitializerImpl::New { init, super_init }) } /// Constructs a new initializer from an initializer for the base class. @@ -245,13 +250,18 @@ impl PyObjectInit for PyClassInitializer { contents: MaybeUninit>, } - let obj = self.super_init.into_new_object(py, subtype)?; + let (init, super_init) = match self.0 { + PyClassInitializerImpl::Existing(value) => return Ok(value.into_ptr()), + PyClassInitializerImpl::New { init, super_init } => (init, super_init), + }; + + let obj = super_init.into_new_object(py, subtype)?; let cell: *mut PartiallyInitializedPyCell = obj as _; std::ptr::write( (*cell).contents.as_mut_ptr(), PyCellContents { - value: ManuallyDrop::new(UnsafeCell::new(self.init)), + value: ManuallyDrop::new(UnsafeCell::new(init)), borrow_checker: ::Storage::new(), thread_checker: T::ThreadChecker::new(), dict: T::Dict::INIT, @@ -287,6 +297,13 @@ where } } +impl From> for PyClassInitializer { + #[inline] + fn from(value: Py) -> PyClassInitializer { + PyClassInitializer(PyClassInitializerImpl::Existing(value)) + } +} + // Implementation used by proc macros to allow anything convertible to PyClassInitializer to be // the return value of pyclass #[new] method (optionally wrapped in `Result`). impl IntoPyCallbackOutput> for U diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index b9b0d152086..ff159c610f8 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -2,6 +2,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use pyo3::sync::GILOnceCell; use pyo3::types::IntoPyDict; #[pyclass] @@ -204,3 +205,62 @@ fn new_with_custom_error() { assert_eq!(err.to_string(), "ValueError: custom error"); }); } + +#[pyclass] +struct NewExisting { + #[pyo3(get)] + num: usize, +} + +#[pymethods] +impl NewExisting { + #[new] + fn new(py: pyo3::Python<'_>, val: usize) -> pyo3::Py { + static PRE_BUILT: GILOnceCell<[pyo3::Py; 2]> = GILOnceCell::new(); + let existing = PRE_BUILT.get_or_init(py, || { + [ + pyo3::PyCell::new(py, NewExisting { num: 0 }) + .unwrap() + .into(), + pyo3::PyCell::new(py, NewExisting { num: 1 }) + .unwrap() + .into(), + ] + }); + + if val < existing.len() { + return existing[val].clone_ref(py); + } + + pyo3::PyCell::new(py, NewExisting { num: val }) + .unwrap() + .into() + } +} + +#[test] +fn test_new_existing() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + + let obj1 = typeobj.call1((0,)).unwrap(); + let obj2 = typeobj.call1((0,)).unwrap(); + let obj3 = typeobj.call1((1,)).unwrap(); + let obj4 = typeobj.call1((1,)).unwrap(); + let obj5 = typeobj.call1((2,)).unwrap(); + let obj6 = typeobj.call1((2,)).unwrap(); + + assert!(obj1.getattr("num").unwrap().extract::().unwrap() == 0); + assert!(obj2.getattr("num").unwrap().extract::().unwrap() == 0); + assert!(obj3.getattr("num").unwrap().extract::().unwrap() == 1); + assert!(obj4.getattr("num").unwrap().extract::().unwrap() == 1); + assert!(obj5.getattr("num").unwrap().extract::().unwrap() == 2); + assert!(obj6.getattr("num").unwrap().extract::().unwrap() == 2); + + assert!(obj1.is(obj2)); + assert!(obj3.is(obj4)); + assert!(!obj1.is(obj3)); + assert!(!obj1.is(obj5)); + assert!(!obj5.is(obj6)); + }); +} From 9151c967d23f697cab0b4b44b6f7af4da59a2adb Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 3 Jul 2023 11:26:45 +0100 Subject: [PATCH 56/59] support PyPy 3.10 --- .github/workflows/build.yml | 2 +- .github/workflows/ci.yml | 2 +- newsfragments/3289.packaging.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 newsfragments/3289.packaging.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5bf947fca13..f1e92149e7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ on: jobs: build: - continue-on-error: ${{ contains(fromJSON('["3.7", "3.12-dev", "pypy3.7", "pypy3.10-nightly"]'), inputs.python-version) }} + continue-on-error: ${{ contains(fromJSON('["3.7", "3.12-dev", "pypy3.7", "pypy3.10"]'), inputs.python-version) }} runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4fcce39c46..4bbd8aa46c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,7 +198,7 @@ jobs: "pypy3.7", "pypy3.8", "pypy3.9", - "pypy3.10-nightly", + "pypy3.10", ] platform: [ diff --git a/newsfragments/3289.packaging.md b/newsfragments/3289.packaging.md new file mode 100644 index 00000000000..a2092bc7903 --- /dev/null +++ b/newsfragments/3289.packaging.md @@ -0,0 +1 @@ +Support PyPy 3.10. From 5fb41a43a0362053fdad388e807f4c4efd134989 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 3 Jul 2023 11:24:08 +0100 Subject: [PATCH 57/59] fixup minimal package versions for msrv --- noxfile.py | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/noxfile.py b/noxfile.py index f332b98f30b..23f04785fed 100644 --- a/noxfile.py +++ b/noxfile.py @@ -444,17 +444,43 @@ def set_minimal_package_versions(session: nox.Session): "examples/word-count", ) min_pkg_versions = { - "rust_decimal": "1.26.1", + # newer versions of rust_decimal want newer arrayvec + "rust_decimal": "1.18.0", + # newer versions of arrayvec use const generics (Rust 1.51+) + "arrayvec": "0.5.2", "csv": "1.1.6", - "hashbrown": "0.12.3", - "once_cell": "1.17.2", - "rayon": "1.6.1", - "rayon-core": "1.10.2", + # newer versions of chrono use i32::rem_euclid as a const fn + "chrono": "0.4.24", + "indexmap": "1.6.2", + "inventory": "0.3.4", + "hashbrown": "0.9.1", + "plotters": "0.3.1", + "plotters-svg": "0.3.1", + "plotters-backend": "0.3.2", + "bumpalo": "3.10.0", + "once_cell": "1.14.0", + "rayon": "1.5.3", + "rayon-core": "1.9.3", "regex": "1.7.3", - "proptest": "1.0.0", - "indexmap": "1.9.3", + # string_cache 0.8.4 depends on parking_lot 0.12 + "string_cache": "0.8.3", + # 1.15.0 depends on hermit-abi 0.2.6 which has edition 2021 and breaks 1.48.0 + "num_cpus": "1.14.0", + "parking_lot": "0.11.0", + # 1.0.77 needs basic-toml which has edition 2021 + "trybuild": "1.0.76", + # pins to avoid syn 2.0 (which requires Rust 1.56) + "ghost": "0.1.8", + "serde": "1.0.156", + "serde_derive": "1.0.156", + "cxx": "1.0.92", + "cxxbridge-macro": "1.0.92", + "cxx-build": "1.0.92", + "web-sys": "0.3.61", + "js-sys": "0.3.61", + "wasm-bindgen": "0.2.84", + "syn": "1.0.109", } - # run cargo update first to ensure that everything is at highest # possible version, so that this matches what CI will resolve to. for project in projects: @@ -626,7 +652,7 @@ def _run_cargo_set_package_version( *, project: Optional[str] = None, ) -> None: - command = ["cargo", "update", "-p", pkg_id, "--precise", version, "--workspace"] + command = ["cargo", "update", "-p", pkg_id, "--precise", version] if project: command.append(f"--manifest-path={project}/Cargo.toml") _run(session, *command, external=True) From 7b2d6276ca5edd5e1a4d5e8b6ae10ceb3d51f745 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 3 Jul 2023 12:07:48 +0100 Subject: [PATCH 58/59] workaround msrv lint issue --- pyo3-ffi/src/cpython/unicodeobject.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index dc5d3ff92f4..1bd93655c6a 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -262,22 +262,19 @@ pub struct PyASCIIObject { /// Interacting with the bitfield is not actually well-defined, so we mark these APIs unsafe. impl PyASCIIObject { - #[cfg_attr(not(Py_3_12), allow(rustdoc::broken_intra_doc_links))] // SSTATE_INTERNED_IMMORTAL_STATIC requires 3.12 /// Get the `interned` field of the [`PyASCIIObject`] state bitfield. /// /// Returns one of: [`SSTATE_NOT_INTERNED`], [`SSTATE_INTERNED_MORTAL`], - /// [`SSTATE_INTERNED_IMMORTAL`], or [`SSTATE_INTERNED_IMMORTAL_STATIC`]. + /// or [`SSTATE_INTERNED_IMMORTAL`]. #[inline] pub unsafe fn interned(&self) -> c_uint { PyASCIIObjectState::from(self.state).interned() } - #[cfg_attr(not(Py_3_12), allow(rustdoc::broken_intra_doc_links))] // SSTATE_INTERNED_IMMORTAL_STATIC requires 3.12 /// Set the `interned` field of the [`PyASCIIObject`] state bitfield. /// /// Calling this function with an argument that is not [`SSTATE_NOT_INTERNED`], - /// [`SSTATE_INTERNED_MORTAL`], [`SSTATE_INTERNED_IMMORTAL`], or - /// [`SSTATE_INTERNED_IMMORTAL_STATIC`] is invalid. + /// [`SSTATE_INTERNED_MORTAL`], or [`SSTATE_INTERNED_IMMORTAL`] is invalid. #[inline] pub unsafe fn set_interned(&mut self, val: c_uint) { let mut state = PyASCIIObjectState::from(self.state); From dfe04e9b316a54c860539bda235972ebcf24a54d Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 3 Jul 2023 10:49:31 +0100 Subject: [PATCH 59/59] release: 0.19.1 --- CHANGELOG.md | 28 ++++++++++++++++++- Cargo.toml | 8 +++--- README.md | 4 +-- examples/Cargo.toml | 2 +- examples/decorator/.template/pre-script.rhai | 2 +- .../maturin-starter/.template/pre-script.rhai | 2 +- examples/plugin/.template/pre-script.rhai | 2 +- .../.template/pre-script.rhai | 2 +- examples/word-count/.template/pre-script.rhai | 2 +- newsfragments/3156.added.md | 1 - newsfragments/3185.fixed.md | 1 - newsfragments/3197.added.md | 1 - newsfragments/3198.added.md | 1 - newsfragments/3248.added.md | 1 - newsfragments/3258.packaging.md | 1 - newsfragments/3263.added.md | 1 - newsfragments/3270.added.md | 1 - newsfragments/3271.fixed.md | 1 - newsfragments/3277.packaging.md | 1 - newsfragments/3286.fixed.md | 1 - newsfragments/3287.added.md | 1 - newsfragments/3289.packaging.md | 1 - pyo3-build-config/Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 4 +-- pyo3-macros-backend/Cargo.toml | 2 +- pyo3-macros/Cargo.toml | 4 +-- pyproject.toml | 2 +- 27 files changed, 46 insertions(+), 33 deletions(-) delete mode 100644 newsfragments/3156.added.md delete mode 100644 newsfragments/3185.fixed.md delete mode 100644 newsfragments/3197.added.md delete mode 100644 newsfragments/3198.added.md delete mode 100644 newsfragments/3248.added.md delete mode 100644 newsfragments/3258.packaging.md delete mode 100644 newsfragments/3263.added.md delete mode 100644 newsfragments/3270.added.md delete mode 100644 newsfragments/3271.fixed.md delete mode 100644 newsfragments/3277.packaging.md delete mode 100644 newsfragments/3286.fixed.md delete mode 100644 newsfragments/3287.added.md delete mode 100644 newsfragments/3289.packaging.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 11c9b6bb772..fe87c5c3ba7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,31 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.19.1] - 2023-07-03 + +### Packaging + +- Extend range of supported versions of `hashbrown` optional dependency to include version 0.14 [#3258](https://github.com/PyO3/pyo3/pull/3258) +- Extend range of supported versions of `indexmap` optional dependency to include version 2. [#3277](https://github.com/PyO3/pyo3/pull/3277) +- Support PyPy 3.10. [#3289](https://github.com/PyO3/pyo3/pull/3289) + +### Added + +- Add `pyo3::types::PyFrozenSetBuilder` to allow building a `PyFrozenSet` item by item. [#3156](https://github.com/PyO3/pyo3/pull/3156) +- Add support for converting to and from Python's `ipaddress.IPv4Address`/`ipaddress.IPv6Address` and `std::net::IpAddr`. [#3197](https://github.com/PyO3/pyo3/pull/3197) +- Add support for `num-bigint` feature in combination with `abi3`. [#3198](https://github.com/PyO3/pyo3/pull/3198) +- Add `PyErr_GetRaisedException()`, `PyErr_SetRaisedException()` to FFI definitions for Python 3.12 and later. [#3248](https://github.com/PyO3/pyo3/pull/3248) +- Add `Python::with_pool` which is a safer but more limited alternative to `Python::new_pool`. [#3263](https://github.com/PyO3/pyo3/pull/3263) +- Add `PyDict::get_item_with_error` on PyPy. [#3270](https://github.com/PyO3/pyo3/pull/3270) +- Allow `#[new]` methods may to return `Py` in order to return existing instances. [#3287](https://github.com/PyO3/pyo3/pull/3287) + +### Fixed + +- Fix conversion of classes implementing `__complex__` to `Complex` when using `abi3` or PyPy. [#3185](https://github.com/PyO3/pyo3/pull/3185) +- Stop suppressing unrelated exceptions in `PyAny::hasattr`. [#3271](https://github.com/PyO3/pyo3/pull/3271) +- Fix memory leak when creating `PySet` or `PyFrozenSet` or returning types converted into these internally, e.g. `HashSet` or `BTreeSet`. [#3286](https://github.com/PyO3/pyo3/pull/3286) + + ## [0.19.0] - 2023-05-31 ### Packaging @@ -1478,7 +1503,8 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.19.0...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.19.1...HEAD +[0.19.1]: https://github.com/pyo3/pyo3/compare/v0.19.0...v0.19.1 [0.19.0]: https://github.com/pyo3/pyo3/compare/v0.18.3...v0.19.0 [0.18.3]: https://github.com/pyo3/pyo3/compare/v0.18.2...v0.18.3 [0.18.2]: https://github.com/pyo3/pyo3/compare/v0.18.1...v0.18.2 diff --git a/Cargo.toml b/Cargo.toml index 115375af7b3..d6dd041bb90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.19.0" +version = "0.19.1" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -20,10 +20,10 @@ parking_lot = ">= 0.11, < 0.13" memoffset = "0.9" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.19.0" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.19.1" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.19.0", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.19.1", optional = true } indoc = { version = "1.0.3", optional = true } unindent = { version = "0.1.4", optional = true } @@ -58,7 +58,7 @@ rust_decimal = { version = "1.8.0", features = ["std"] } widestring = "0.5.1" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "0.19.0", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "0.19.1", features = ["resolve-config"] } [features] default = ["macros"] diff --git a/README.md b/README.md index 5b25101bf20..cc45f93e29c 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.19.0", features = ["extension-module"] } +pyo3 = { version = "0.19.1", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.19.0" +version = "0.19.1" features = ["auto-initialize"] ``` diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 43907a8becd..bdc78d6a034 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -5,7 +5,7 @@ publish = false edition = "2018" [dev-dependencies] -pyo3 = { version = "0.19.0", path = "..", features = ["auto-initialize", "extension-module"] } +pyo3 = { version = "0.19.1", path = "..", features = ["auto-initialize", "extension-module"] } [[example]] name = "decorator" diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 0368bb1f432..088ea73bfbe 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 0368bb1f432..088ea73bfbe 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 0915e77badd..158e1522040 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 3ed1b01d30f..19ea7cc8520 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 68988ce3f48..a4bfa7ce13b 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,3 +1,3 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::delete(".template"); diff --git a/newsfragments/3156.added.md b/newsfragments/3156.added.md deleted file mode 100644 index 75a83ef0fe8..00000000000 --- a/newsfragments/3156.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `pyo3::types::PyFrozenSetBuilder` to allow building a `PyFrozenSet` item by item. \ No newline at end of file diff --git a/newsfragments/3185.fixed.md b/newsfragments/3185.fixed.md deleted file mode 100644 index 6b3a594c91a..00000000000 --- a/newsfragments/3185.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix conversion of classes implementing `__complex__` to `Complex` when using `abi3` or PyPy. diff --git a/newsfragments/3197.added.md b/newsfragments/3197.added.md deleted file mode 100644 index eb8affb1042..00000000000 --- a/newsfragments/3197.added.md +++ /dev/null @@ -1 +0,0 @@ -Add support for converting to and from Python's `ipaddress.IPv4Address`/`ipaddress.IPv6Address` and `std::net::IpAddr`. diff --git a/newsfragments/3198.added.md b/newsfragments/3198.added.md deleted file mode 100644 index 2c6514a42f8..00000000000 --- a/newsfragments/3198.added.md +++ /dev/null @@ -1 +0,0 @@ -Add support for `num-bigint` feature in combination with `abi3`. \ No newline at end of file diff --git a/newsfragments/3248.added.md b/newsfragments/3248.added.md deleted file mode 100644 index 5fb5656f753..00000000000 --- a/newsfragments/3248.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyErr_GetRaisedException()`, `PyErr_SetRaisedException()` to FFI definitions for Python 3.12 and later. diff --git a/newsfragments/3258.packaging.md b/newsfragments/3258.packaging.md deleted file mode 100644 index 758e54584d9..00000000000 --- a/newsfragments/3258.packaging.md +++ /dev/null @@ -1 +0,0 @@ -Enabled support for hashbrown version 0.14 diff --git a/newsfragments/3263.added.md b/newsfragments/3263.added.md deleted file mode 100644 index 888395d61f6..00000000000 --- a/newsfragments/3263.added.md +++ /dev/null @@ -1 +0,0 @@ -Add the `Python::with_pool` which is a safer but more limited alternative to `Python::new_pool`. diff --git a/newsfragments/3270.added.md b/newsfragments/3270.added.md deleted file mode 100644 index 66b2db9d83d..00000000000 --- a/newsfragments/3270.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyDict::get_item_with_error` on PyPy. diff --git a/newsfragments/3271.fixed.md b/newsfragments/3271.fixed.md deleted file mode 100644 index ff3fe71517f..00000000000 --- a/newsfragments/3271.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Stop suppressing unrelated exceptions in `PyAny::hasattr`. diff --git a/newsfragments/3277.packaging.md b/newsfragments/3277.packaging.md deleted file mode 100644 index 1fbed18467d..00000000000 --- a/newsfragments/3277.packaging.md +++ /dev/null @@ -1 +0,0 @@ -Extended range of supported versions of `indexmap` optional dependency to include version 2. diff --git a/newsfragments/3286.fixed.md b/newsfragments/3286.fixed.md deleted file mode 100644 index cfe647b9eb0..00000000000 --- a/newsfragments/3286.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak when creating `PySet` or `PyFrozenSet` or returning types converted into these internally, e.g. `HashSet` or `BTreeSet`. diff --git a/newsfragments/3287.added.md b/newsfragments/3287.added.md deleted file mode 100644 index bde61a4b506..00000000000 --- a/newsfragments/3287.added.md +++ /dev/null @@ -1 +0,0 @@ -`#[new]` methods may now return `Py` in order to return existing instances diff --git a/newsfragments/3289.packaging.md b/newsfragments/3289.packaging.md deleted file mode 100644 index a2092bc7903..00000000000 --- a/newsfragments/3289.packaging.md +++ /dev/null @@ -1 +0,0 @@ -Support PyPy 3.10. diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index ac8d451fada..50445fc938a 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.19.0" +version = "0.19.1" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index bda76ab6b55..4e910d3fa5a 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.19.0" +version = "0.19.1" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,4 +38,4 @@ generate-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "0.19.0", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "0.19.1", features = ["resolve-config"] } diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index e08919cfe8d..4644fc6e764 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.19.0" +version = "0.19.1" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 6c531c0fecf..347b8d78ae5 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.19.0" +version = "0.19.1" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -22,4 +22,4 @@ abi3 = ["pyo3-macros-backend/abi3"] proc-macro2 = { version = "1", default-features = false } quote = "1" syn = { version = "1.0.85", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.19.0" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.19.1" } diff --git a/pyproject.toml b/pyproject.toml index f53001e2f93..ae99ebf75f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ exclude = ''' [tool.towncrier] filename = "CHANGELOG.md" -version = "0.19.0" +version = "0.19.1" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}"