gh-103509: PEP 697 -- Limited C API for Extending Opaque Types (GH-103511)

Co-authored-by: Oleg Iarygin <oleg@arhadthedev.net>
Co-authored-by: Erlend E. Aasland <erlend.aasland@protonmail.com>

Author: 3 people

Doc/c-api/object.rst

@@ -395,3 +395,42 @@ Object Protocol
    returns ``NULL`` if the object cannot be iterated.
 
    .. versionadded:: 3.10
+
+.. c:function:: void *PyObject_GetTypeData(PyObject *o, PyTypeObject *cls)
+
+   Get a pointer to subclass-specific data reserved for *cls*.
+
+   The object *o* must be an instance of *cls*, and *cls* must have been
+   created using negative :c:member:`PyType_Spec.basicsize`.
+   Python does not check this.
+
+   On error, set an exception and return ``NULL``.
+
+   .. versionadded:: 3.12
+
+.. c:function:: Py_ssize_t PyType_GetTypeDataSize(PyTypeObject *cls)
+
+   Return the size of the instance memory space reserved for *cls*, i.e. the size of the
+   memory :c:func:`PyObject_GetTypeData` returns.
+
+   This may be larger than requested using :c:member:`-PyType_Spec.basicsize <PyType_Spec.basicsize>`;
+   it is safe to use this larger size (e.g. with :c:func:`!memset`).
+
+   The type *cls* **must** have been created using
+   negative :c:member:`PyType_Spec.basicsize`.
+   Python does not check this.
+
+   On error, set an exception and return a negative value.
+
+   .. versionadded:: 3.12
+
+.. c:function:: void *PyObject_GetItemData(PyObject *o)
+
+   Get a pointer to per-item data for a class with
+   :const:`Py_TPFLAGS_ITEMS_AT_END`.
+
+   On error, set an exception and return ``NULL``.
+   :py:exc:`TypeError` is raised if *o* does not have
+   :const:`Py_TPFLAGS_ITEMS_AT_END` set.
+
+   .. versionadded:: 3.12

Doc/c-api/structures.rst

@@ -486,6 +486,22 @@ The following flags can be used with :c:member:`PyMemberDef.flags`:
    Emit an ``object.__getattr__`` :ref:`audit event <audit-events>`
    before reading.
 
+.. c:macro:: Py_RELATIVE_OFFSET
+
+   Indicates that the :c:member:`~PyMemberDef.offset` of this ``PyMemberDef``
+   entry indicates an offset from the subclass-specific data, rather than
+   from ``PyObject``.
+
+   Can only be used as part of :c:member:`Py_tp_members <PyTypeObject.tp_members>`
+   :c:type:`slot <PyTypeSlot>` when creating a class using negative
+   :c:member:`~PyTypeDef.basicsize`.
+   It is mandatory in that case.
+
+   This flag is only used in :c:type:`PyTypeSlot`.
+   When setting :c:member:`~PyTypeObject.tp_members` during
+   class creation, Python clears it and sets
+   :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct.
+
 .. index::
    single: READ_RESTRICTED
    single: WRITE_RESTRICTED

Doc/c-api/type.rst

@@ -353,25 +353,57 @@ The following functions and structs are used to create
 
    Structure defining a type's behavior.
 
-   .. c:member:: const char* PyType_Spec.name
+   .. c:member:: const char* name
 
       Name of the type, used to set :c:member:`PyTypeObject.tp_name`.
 
-   .. c:member:: int PyType_Spec.basicsize
-   .. c:member:: int PyType_Spec.itemsize
+   .. c:member:: int basicsize
 
-      Size of the instance in bytes, used to set
-      :c:member:`PyTypeObject.tp_basicsize` and
-      :c:member:`PyTypeObject.tp_itemsize`.
+      If positive, specifies the size of the instance in bytes.
+      It is used to set :c:member:`PyTypeObject.tp_basicsize`.
 
-   .. c:member:: int PyType_Spec.flags
+      If zero, specifies that :c:member:`~PyTypeObject.tp_basicsize`
+      should be inherited.
+
+      If negative, the absolute value specifies how much space instances of the
+      class need *in addition* to the superclass.
+      Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific
+      memory reserved this way.
+
+      .. versionchanged:: 3.12
+
+         Previously, this field could not be negative.
+
+   .. c:member:: int itemsize
+
+      Size of one element of a variable-size type, in bytes.
+      Used to set :c:member:`PyTypeObject.tp_itemsize`.
+      See ``tp_itemsize`` documentation for caveats.
+
+      If zero, :c:member:`~PyTypeObject.tp_itemsize` is inherited.
+      Extending arbitrary variable-sized classes is dangerous,
+      since some types use a fixed offset for variable-sized memory,
+      which can then overlap fixed-sized memory used by a subclass.
+      To help prevent mistakes, inheriting ``itemsize`` is only possible
+      in the following situations:
+
+      - The base is not variable-sized (its
+        :c:member:`~PyTypeObject.tp_itemsize`).
+      - The requested :c:member:`PyType_Spec.basicsize` is positive,
+        suggesting that the memory layout of the base class is known.
+      - The requested :c:member:`PyType_Spec.basicsize` is zero,
+        suggesting that the subclass does not access the instance's memory
+        directly.
+      - With the :const:`Py_TPFLAGS_ITEMS_AT_END` flag.
+
+   .. c:member:: unsigned int flags
 
       Type flags, used to set :c:member:`PyTypeObject.tp_flags`.
 
       If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set,
       :c:func:`PyType_FromSpecWithBases` sets it automatically.
 
-   .. c:member:: PyType_Slot *PyType_Spec.slots
+   .. c:member:: PyType_Slot *slots
 
       Array of :c:type:`PyType_Slot` structures.
       Terminated by the special slot value ``{0, NULL}``.

Doc/c-api/typeobj.rst

@@ -1171,6 +1171,26 @@ and :c:type:`PyType_Type` effectively act as defaults.)
       :c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass.
 
 
+   .. data:: Py_TPFLAGS_ITEMS_AT_END
+
+      Only usable with variable-size types, i.e. ones with non-zero
+      :c:member:`~PyObject.tp_itemsize`.
+
+      Indicates that the variable-sized portion of an instance of this type is
+      at the end of the instance's memory area, at an offset of
+      :c:expr:`Py_TYPE(obj)->tp_basicsize` (which may be different in each
+      subclass).
+
+      When setting this flag, be sure that all superclasses either
+      use this memory layout, or are not variable-sized.
+      Python does not check this.
+
+      .. versionadded:: 3.12
+
+      **Inheritance:**
+
+      This flag is inherited.
+
    .. XXX Document more flags here?
 
 

Doc/data/stable_abi.dat

@@ -1159,6 +1159,21 @@ New Features
 
   (Contributed by Petr Viktorin in :gh:`101101`.)
 
+* :pep:`697`: Added API for extending types whose instance memory layout is
+  opaque:
+
+  - :c:member:`PyType_Spec.basicsize` can be zero or negative to specify
+    inheriting or extending the base class size.
+  - :c:func:`PyObject_GetTypeData` and :c:func:`PyType_GetTypeDataSize`
+    added to allow access to subclass-specific instance data.
+  - :const:`Py_TPFLAGS_ITEMS_AT_END` and :c:func:`PyObject_GetItemData`
+    added to allow safely extending certain variable-sized types, including
+    :c:var:`PyType_Type`.
+  - :c:macro:`Py_RELATIVE_OFFSET` added to allow defining
+    :c:type:`members <PyMemberDef>` in terms of a subclass-specific struct.
+
+  (Contributed by Petr Viktorin in :gh:`103509`.)
+
 * Added the new limited C API function :c:func:`PyType_FromMetaclass`,
   which generalizes the existing :c:func:`PyType_FromModuleAndSpec` using
   an additional metaclass argument.

Doc/whatsnew/3.12.rst

@@ -553,6 +553,7 @@ Py_DEPRECATED(3.11) typedef int UsingDeprecatedTrashcanMacro;
         Py_TRASHCAN_END; \
     } while(0);
 
+PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj);
 
 PyAPI_FUNC(int) _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg);
 PyAPI_FUNC(void) _PyObject_ClearManagedDict(PyObject *obj);

Include/cpython/object.h

@@ -83,6 +83,7 @@ struct PyMemberDef {
 #define Py_READONLY            1
 #define Py_AUDIT_READ          2 // Added in 3.10, harmless no-op before that
 #define _Py_WRITE_RESTRICTED   4 // Deprecated, no-op. Do not reuse the value.
+#define Py_RELATIVE_OFFSET     8
 
 PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, PyMemberDef *);
 PyAPI_FUNC(int) PyMember_SetOne(char *, PyMemberDef *, PyObject *);

Include/descrobject.h

@@ -389,11 +389,6 @@ extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);
 extern void _PyObject_FreeInstanceAttributes(PyObject *obj);
 extern int _PyObject_IsInstanceDictEmpty(PyObject *);
 
-// Access macro to the members which are floating "behind" the object
-static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) {
-    return (PyMemberDef*)((char*)etype + Py_TYPE(etype)->tp_basicsize);
-}
-
 PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, PyObject *);
 
 /* C function call trampolines to mitigate bad function pointer casts.

Include/internal/pycore_object.h

@@ -355,6 +355,8 @@ PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *);
 #endif
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000
 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*);
+PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls);
+PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls);
 #endif
 
 /* Generic type check */
@@ -521,6 +523,9 @@ given type object has a specified feature.
 // subject itself (rather than a mapped attribute on it):
 #define _Py_TPFLAGS_MATCH_SELF (1UL << 22)
 
+/* Items (ob_size*tp_itemsize) are found at the end of an instance's memory */
+#define Py_TPFLAGS_ITEMS_AT_END (1UL << 23)
+
 /* These flags are used to determine if a type is a subclass. */
 #define Py_TPFLAGS_LONG_SUBCLASS        (1UL << 24)
 #define Py_TPFLAGS_LIST_SUBCLASS        (1UL << 25)