pymethods: add support for sequence protocol

guide/src/migration.md

@@ -5,10 +5,42 @@ For a detailed list of all changes, see the [CHANGELOG](changelog.md).
 
 ## from 0.15.* to 0.16
 
-### Drop support for older technogies
+### Drop support for older technologies
 
 PyO3 0.16 has increased minimum Rust version to 1.48 and minimum Python version to 3.7. This enables ore use of newer language features (enabling some of the other additions in 0.16) and simplifies maintenance of the project.
 
+### Container magic methods now match Python behavior
+
+In PyO3 0.15, `__len__`, `__getitem__`, `__setitem__` and `__delitem__` in `#[pymethods]` would generate only the _mapping_ implementation for a `#[pyclass]`. To match the Python behavior, these methods now generate both the _mapping_ **and** _sequence_ implementations.
+
+This means that classes implementing these `#[pymethods]` will now also be treated as sequences, same as a Python `class` would be. Small differences in behavior may result:
+ - PyO3 will allow instances of these classes to be cast to `PySequence` as well as `PyMapping`.
+ - Python will provide a default implementation of `__iter__` (if the class did not have one) which repeatedly calls `__getitem__` with integers (starting at 0) until an `IndexError` is raised.
+
+To disable this behavior, use `#[pyclass(true_mapping)]` to retain the previous behavior of only providing the mapping implementation.
+
+To explain this in detail, consider the following Python class:
+
+```python
+class ExampleContainer:
+
+ def __len__(self):
+ return 5
+
+ def __getitem__(self, idx: int) -> int:
+ if idx < 0 or idx > 5:
+ raise IndexError()
+ return idx
+```
+
+This class implements a Python [sequence](https://docs.python.org/3/glossary.html#term-sequence).
+
+The `__len__` and `__getitem__` methods are also used to implement a Python [mapping](https://docs.python.org/3/glossary.html#term-mapping). In the Python C-API, these methods are not shared: the sequence `__len__` and `__getitem__` are defined by the `sq_len` and `sq_item` slots, and the mapping equivalents are `mp_len` and `mp_subscript`. There are similar distinctions for `__setitem__` and `__delitem__`.
+
+Because there is no such distinction from Python, implementing these methods will fill the mapping and sequence slots simultaneously. A Python class with `__len__` implemented, for example, will have both the `sq_len` and `mp_len` slots filled.
+
+The PyO3 behavior in 0.16 has been changed to match this Python behavior by default. PyO3 also provides ways to implement a pure mapping using the `#[true_mapping]` attribute and a pure sequence using the PyO3-specific `__seqlen__`, `__getseqitem__`, `__setseqitem__`, and `__delseqitem__` methods.
+
 ## from 0.14.* to 0.15
 
 ### Changes in sequence indexing

pyo3-macros-backend/src/pyclass.rs

@@ -35,6 +35,7 @@ pub struct PyClassArgs {
  pub has_unsendable: bool,
  pub module: Option<syn::LitStr>,
  pub class_kind: PyClassKind,
+ pub true_mapping: bool,
 }
 
 impl PyClassArgs {
@@ -68,6 +69,7 @@ impl PyClassArgs {
  has_extends: false,
  has_unsendable: false,
  class_kind,
+ true_mapping: false,
  }
  }
 
@@ -176,8 +178,11 @@ impl PyClassArgs {
  "unsendable" => {
  self.has_unsendable = true;
  }
+ "true_mapping" => {
+ self.true_mapping = true;
+ }
  _ => bail_spanned!(
- exp.path.span() => "expected one of gc/weakref/subclass/dict/unsendable"
+ exp.path.span() => "expected one of gc/weakref/subclass/dict/unsendable/true_mapping"
  ),
  };
  Ok(())
@@ -730,6 +735,7 @@ impl<'a> PyClassImplsBuilder<'a> {
  let is_basetype = self.attr.is_basetype;
  let base = &self.attr.base;
  let is_subclass = self.attr.has_extends;
+ let true_mapping = self.attr.true_mapping;
 
  let thread_checker = if self.attr.has_unsendable {
  quote! { _pyo3::class::impl_::ThreadCheckerImpl<#cls> }
@@ -778,6 +784,7 @@ impl<'a> PyClassImplsBuilder<'a> {
  const IS_GC: bool = #is_gc;
  const IS_BASETYPE: bool = #is_basetype;
  const IS_SUBCLASS: bool = #is_subclass;
+ const TRUE_MAPPING: bool = #true_mapping;
 
  type Layout = _pyo3::PyCell<Self>;
  type BaseType = #base;

pyo3-macros-backend/src/pyimpl.rs

@@ -274,6 +274,11 @@ fn add_shared_proto_slots(
  try_add_shared_slot!("__setattr__", "__delattr__", generate_pyclass_setattr_slot);
  try_add_shared_slot!("__set__", "__delete__", generate_pyclass_setdescr_slot);
  try_add_shared_slot!("__setitem__", "__delitem__", generate_pyclass_setitem_slot);
+ try_add_shared_slot!(
+ "__setseqitem__",
+ "__delseqitem__",
+ generate_pyclass_setseqitem_slot
+ );
  try_add_shared_slot!("__add__", "__radd__", generate_pyclass_add_slot);
  try_add_shared_slot!("__sub__", "__rsub__", generate_pyclass_sub_slot);
  try_add_shared_slot!("__mul__", "__rmul__", generate_pyclass_mul_slot);
@@ -293,6 +298,8 @@ fn add_shared_proto_slots(
  );
  try_add_shared_slot!("__pow__", "__rpow__", generate_pyclass_pow_slot);
 
+ // if this assertion trips, a slot fragment has been implemented which has not been added in the
+ // list above
  assert!(implemented_proto_fragments.is_empty());
 }
 

pyo3-macros-backend/src/pymethod.rs

@@ -490,10 +490,13 @@ const __ANEXT__: SlotDef = SlotDef::new("Py_am_anext", "unaryfunc").return_conve
  TokenGenerator(|| quote! { _pyo3::class::pyasync::IterANextOutput::<_, _> }),
 );
 const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc").ret_ty(Ty::PySsizeT);
+const __SEQLEN__: SlotDef = SlotDef::new("Py_sq_length", "lenfunc").ret_ty(Ty::PySsizeT);
 const __CONTAINS__: SlotDef = SlotDef::new("Py_sq_contains", "objobjproc")
  .arguments(&[Ty::Object])
  .ret_ty(Ty::Int);
 const __GETITEM__: SlotDef = SlotDef::new("Py_mp_subscript", "binaryfunc").arguments(&[Ty::Object]);
+const __GETSEQITEM__: SlotDef =
+ SlotDef::new("Py_sq_item", "ssizeargfunc").arguments(&[Ty::PySsizeT]);
 
 const __POS__: SlotDef = SlotDef::new("Py_nb_positive", "unaryfunc");
 const __NEG__: SlotDef = SlotDef::new("Py_nb_negative", "unaryfunc");
@@ -571,8 +574,10 @@ fn pyproto(method_name: &str) -> Option<&'static SlotDef> {
  "__aiter__" => Some(&__AITER__),
  "__anext__" => Some(&__ANEXT__),
  "__len__" => Some(&__LEN__),
+ "__seqlen__" => Some(&__SEQLEN__),
  "__contains__" => Some(&__CONTAINS__),
  "__getitem__" => Some(&__GETITEM__),
+ "__getseqitem__" => Some(&__GETSEQITEM__),
  "__pos__" => Some(&__POS__),
  "__neg__" => Some(&__NEG__),
  "__abs__" => Some(&__ABS__),
@@ -680,7 +685,22 @@ impl Ty {
  let #ident = #extract;
  }
  }
- Ty::Int | Ty::PyHashT | Ty::PySsizeT | Ty::Void => todo!(),
+ Ty::PySsizeT => {
+ let ty = arg.ty;
+ let extract = handle_error(
+ extract_error_mode,
+ py,
+ quote! {
+ ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| _pyo3::exceptions::PyValueError::new_err(e.to_string()))
+ },
+ );
+ quote! {
+ let #ident = #extract;
+ }
+ }
+ Ty::Int | Ty::PyHashT | Ty::Void => {
+ unimplemented!("not used as magic method arguments")
+ }
  }
  }
 }
@@ -937,6 +957,9 @@ const __DELETE__: SlotFragmentDef = SlotFragmentDef::new("__delete__", &[Ty::Obj
 const __SETITEM__: SlotFragmentDef =
  SlotFragmentDef::new("__setitem__", &[Ty::Object, Ty::NonNullObject]);
 const __DELITEM__: SlotFragmentDef = SlotFragmentDef::new("__delitem__", &[Ty::Object]);
+const __SETSEQITEM__: SlotFragmentDef =
+ SlotFragmentDef::new("__setseqitem__", &[Ty::PySsizeT, Ty::NonNullObject]);
+const __DELSEQITEM__: SlotFragmentDef = SlotFragmentDef::new("__delseqitem__", &[Ty::PySsizeT]);
 
 macro_rules! binary_num_slot_fragment_def {
  ($ident:ident, $name:literal) => {
@@ -988,6 +1011,8 @@ fn pyproto_fragment(method_name: &str) -> Option<&'static SlotFragmentDef> {
  "__delete__" => Some(&__DELETE__),
  "__setitem__" => Some(&__SETITEM__),
  "__delitem__" => Some(&__DELITEM__),
+ "__setseqitem__" => Some(&__SETSEQITEM__),
+ "__delseqitem__" => Some(&__DELSEQITEM__),
  "__add__" => Some(&__ADD__),
  "__radd__" => Some(&__RADD__),
  "__sub__" => Some(&__SUB__),

src/class/impl_.rs

@@ -9,7 +9,12 @@ use crate::{
  type_object::{PyLayout, PyTypeObject},
  PyClass, PyMethodDefType, PyNativeType, PyResult, PyTypeInfo, Python,
 };
-use std::{marker::PhantomData, os::raw::c_void, ptr::NonNull, thread};
+use std::{
+ marker::PhantomData,
+ os::raw::{c_int, c_void},
+ ptr::NonNull,
+ thread,
+};
 
 /// This type is used as a "dummy" type on which dtolnay specializations are
 /// applied to apply implementations from `#[pymethods]` & `#[pyproto]`
@@ -52,6 +57,9 @@ pub trait PyClassImpl: Sized {
  /// #[pyclass(extends=...)]
  const IS_SUBCLASS: bool = false;
 
+ /// #[pyclass(true_mapping)]
+ const TRUE_MAPPING: bool = false;
+
  /// Layout
  type Layout: PyLayout<Self>;
 
@@ -223,6 +231,66 @@ define_pyclass_setattr_slot! {
  objobjargproc,
 }
 
+slot_fragment_trait! {
+ PyClass__setseqitem__SlotFragment,
+
+ /// # Safety: _slf and _attr must be valid non-null Python objects
+ #[inline]
+ unsafe fn __setseqitem__(
+ self,
+ _py: Python,
+ _slf: *mut ffi::PyObject,
+ _index: ffi::Py_ssize_t,
+ _value: NonNull<ffi::PyObject>,
+ ) -> PyResult<()> {
+ Err(PyNotImplementedError::new_err("can't set item"))
+ }
+}
+
+slot_fragment_trait! {
+ PyClass__delseqitem__SlotFragment,
+
+ /// # Safety: _slf and _attr must be valid non-null Python objects
+ #[inline]
+ unsafe fn __delseqitem__(
+ self,
+ _py: Python,
+ _slf: *mut ffi::PyObject,
+ _index: ffi::Py_ssize_t,
+ ) -> PyResult<()> {
+ Err(PyNotImplementedError::new_err("can't delete item"))
+ }
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! generate_pyclass_setseqitem_slot {
+ ($cls:ty) => {{
+ unsafe extern "C" fn __wrap(
+ _slf: *mut $crate::ffi::PyObject,
+ index: ffi::Py_ssize_t,
+ value: *mut $crate::ffi::PyObject,
+ ) -> ::std::os::raw::c_int {
+ use ::std::option::Option::*;
+ use $crate::callback::IntoPyCallbackOutput;
+ use $crate::class::impl_::*;
+ $crate::callback::handle_panic(|py| {
+ let collector = PyClassImplCollector::<$cls>::new();
+ if let Some(value) = ::std::ptr::NonNull::new(value) {
+ collector.__setseqitem__(py, _slf, index, value).convert(py)
+ } else {
+ collector.__delseqitem__(py, _slf, index).convert(py)
+ }
+ })
+ }
+ $crate::ffi::PyType_Slot {
+ slot: $crate::ffi::Py_sq_ass_item,
+ pfunc: __wrap as $crate::ffi::ssizeobjargproc as _,
+ }
+ }};
+}
+pub use generate_pyclass_setseqitem_slot;
+
 /// Macro which expands to three items
 /// - Trait for a lhs dunder e.g. __add__
 /// - Trait for the corresponding rhs e.g. __radd__
@@ -795,3 +863,38 @@ pub(crate) unsafe extern "C" fn fallback_new(
 pub(crate) unsafe extern "C" fn tp_dealloc<T: PyClass>(obj: *mut ffi::PyObject) {
  crate::callback_body!(py, T::Layout::tp_dealloc(obj, py))
 }
+
+pub(crate) unsafe extern "C" fn sq_length_from_mapping(obj: *mut ffi::PyObject) -> ffi::Py_ssize_t {
+ ffi::PyMapping_Length(obj)
+}
+
+pub(crate) unsafe extern "C" fn sq_item_from_mapping(
+ obj: *mut ffi::PyObject,
+ index: ffi::Py_ssize_t,
+) -> *mut ffi::PyObject {
+ let index = ffi::PyLong_FromSsize_t(index);
+ if index.is_null() {
+ return std::ptr::null_mut();
+ }
+ let result = ffi::PyObject_GetItem(obj, index);
+ ffi::Py_DECREF(index);
+ result
+}
+
+pub(crate) unsafe extern "C" fn sq_ass_item_from_mapping(
+ obj: *mut ffi::PyObject,
+ index: ffi::Py_ssize_t,
+ value: *mut ffi::PyObject,
+) -> c_int {
+ let index = ffi::PyLong_FromSsize_t(index);
+ if index.is_null() {
+ return -1;
+ }
+ let result = if value.is_null() {
+ ffi::PyObject_DelItem(obj, index)
+ } else {
+ ffi::PyObject_SetItem(obj, index, value)
+ };
+ ffi::Py_DECREF(index);
+ result
+}