gh-60074: add new stable API function PyType_FromMetaclass (GH-93012)

Added a new stable API function ``PyType_FromMetaclass``, which mirrors
the behavior of ``PyType_FromModuleAndSpec`` except that it takes an
additional metaclass argument. This is, e.g., useful for language
binding tools that need to store additional information in the type
object.

Author: wjakob

Doc/c-api/type.rst

@@ -190,11 +190,16 @@ Creating Heap-Allocated Types
 The following functions and structs are used to create
 :ref:`heap types <heap-types>`.
 
-.. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
+.. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases)
 
-   Creates and returns a :ref:`heap type <heap-types>` from the *spec*
+   Create and return a :ref:`heap type <heap-types>` from the *spec*
    (:const:`Py_TPFLAGS_HEAPTYPE`).
 
+   The metaclass *metaclass* is used to construct the resulting type object.
+   When *metaclass* is ``NULL``, the default :c:type:`PyType_Type` is used
+   instead. Note that metaclasses that override
+   :c:member:`~PyTypeObject.tp_new` are not supported.
+
    The *bases* argument can be used to specify base classes; it can either
    be only one class or a tuple of classes.
    If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead.
@@ -210,22 +215,29 @@ The following functions and structs are used to create
 
    This function calls :c:func:`PyType_Ready` on the new type.
 
+   .. versionadded:: 3.12
+
+.. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
+
+   Equivalent to ``PyType_FromMetaclass(NULL, module, spec, bases)``.
+
    .. versionadded:: 3.9
 
    .. versionchanged:: 3.10
 
       The function now accepts a single class as the *bases* argument and
       ``NULL`` as the ``tp_doc`` slot.
 
+
 .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
 
-   Equivalent to ``PyType_FromModuleAndSpec(NULL, spec, bases)``.
+   Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, bases)``.
 
    .. versionadded:: 3.3
 
 .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)
 
-   Equivalent to ``PyType_FromSpecWithBases(spec, NULL)``.
+   Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``.
 
 .. c:type:: PyType_Spec
 

Doc/c-api/typeobj.rst

@@ -2071,7 +2071,7 @@ flag set.
 
 This is done by filling a :c:type:`PyType_Spec` structure and calling
 :c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases`,
-or :c:func:`PyType_FromModuleAndSpec`.
+:c:func:`PyType_FromModuleAndSpec`, or :c:func:`PyType_FromMetaclass`.
 
 
 .. _number-structs:

Doc/data/stable_abi.dat

@@ -151,6 +151,11 @@ C API Changes
 New Features
 ------------
 
+* Added the new limited C API function :c:func:`PyType_FromMetaclass`,
+  which generalizes the existing :c:func:`PyType_FromModuleAndSpec` using
+  an additional metaclass argument.
+  (Contributed by Wenzel Jakob in :gh:`93012`.)
+
 Porting to Python 3.12
 ----------------------
 

Doc/whatsnew/3.12.rst

@@ -257,6 +257,9 @@ PyAPI_FUNC(void *) PyType_GetModuleState(PyTypeObject *);
 PyAPI_FUNC(PyObject *) PyType_GetName(PyTypeObject *);
 PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *);
 #endif
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000
+PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*);
+#endif
 
 /* Generic type check */
 PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *);

Include/object.h

@@ -605,6 +605,19 @@ def test_heaptype_with_setattro(self):
         del obj.value
         self.assertEqual(obj.pvalue, 0)
 
+    def test_heaptype_with_custom_metaclass(self):
+        self.assertTrue(issubclass(_testcapi.HeapCTypeMetaclass, type))
+        self.assertTrue(issubclass(_testcapi.HeapCTypeMetaclassCustomNew, type))
+
+        t = _testcapi.pytype_fromspec_meta(_testcapi.HeapCTypeMetaclass)
+        self.assertIsInstance(t, type)
+        self.assertEqual(t.__name__, "HeapCTypeViaMetaclass")
+        self.assertIs(type(t), _testcapi.HeapCTypeMetaclass)
+
+        msg = "Metaclasses with custom tp_new are not supported."
+        with self.assertRaisesRegex(TypeError, msg):
+            t = _testcapi.pytype_fromspec_meta(_testcapi.HeapCTypeMetaclassCustomNew)
+
     def test_pynumber_tobase(self):
         from _testcapi import pynumber_tobase
         self.assertEqual(pynumber_tobase(123, 2), '0b1111011')

Lib/test/test_capi.py

@@ -0,0 +1,8 @@
+Added the new function :c:func:`PyType_FromMetaclass`, which generalizes the
+existing :c:func:`PyType_FromModuleAndSpec` using an additional metaclass
+argument. This is useful for language binding tools, where it can be used to
+intercept type-related operations like subclassing or static attribute access
+by specifying a metaclass with custom slots.
+
+Importantly, :c:func:`PyType_FromMetaclass` is available in the Limited API,
+which provides a path towards migrating more binding tools onto the Stable ABI.

Lib/test/test_stable_abi_ctypes.py

@@ -2275,3 +2275,5 @@
     added = '3.11'
 [function.PyErr_SetHandledException]
     added = '3.11'
+[function.PyType_FromMetaclass]
+    added = '3.12'

Misc/NEWS.d/next/Core and Builtins/2022-05-20-13-32-24.gh-issue-93012.e9B-pv.rst

@@ -308,6 +308,32 @@ test_dict_inner(int count)
     }
 }
 
+static PyObject *pytype_fromspec_meta(PyObject* self, PyObject *meta)
+{
+    if (!PyType_Check(meta)) {
+        PyErr_SetString(
+            TestError,
+            "pytype_fromspec_meta: must be invoked with a type argument!");
+        return NULL;
+    }
+
+    PyType_Slot HeapCTypeViaMetaclass_slots[] = {
+        {0},
+    };
+
+    PyType_Spec HeapCTypeViaMetaclass_spec = {
+        "_testcapi.HeapCTypeViaMetaclass",
+        sizeof(PyObject),
+        0,
+        Py_TPFLAGS_DEFAULT,
+        HeapCTypeViaMetaclass_slots
+    };
+
+    return PyType_FromMetaclass(
+        (PyTypeObject *) meta, NULL, &HeapCTypeViaMetaclass_spec, NULL);
+}
+
+
 static PyObject*
 test_dict_iteration(PyObject* self, PyObject *Py_UNUSED(ignored))
 {
@@ -5886,6 +5912,7 @@ static PyMethodDef TestMethods[] = {
     {"test_long_numbits",       test_long_numbits,               METH_NOARGS},
     {"test_k_code",             test_k_code,                     METH_NOARGS},
     {"test_empty_argparse",     test_empty_argparse,             METH_NOARGS},
+    {"pytype_fromspec_meta",    pytype_fromspec_meta,            METH_O},
     {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS},
     {"pyobject_repr_from_null", pyobject_repr_from_null, METH_NOARGS},
     {"pyobject_str_from_null",  pyobject_str_from_null, METH_NOARGS},
@@ -7078,6 +7105,38 @@ static PyType_Spec HeapCTypeSubclassWithFinalizer_spec = {
     HeapCTypeSubclassWithFinalizer_slots
 };
 
+static PyType_Slot HeapCTypeMetaclass_slots[] = {
+    {0},
+};
+
+static PyType_Spec HeapCTypeMetaclass_spec = {
+    "_testcapi.HeapCTypeMetaclass",
+    sizeof(PyHeapTypeObject),
+    sizeof(PyMemberDef),
+    Py_TPFLAGS_DEFAULT,
+    HeapCTypeMetaclass_slots
+};
+
+static PyObject *
+heap_ctype_metaclass_custom_tp_new(PyTypeObject *tp, PyObject *args, PyObject *kwargs)
+{
+    return PyType_Type.tp_new(tp, args, kwargs);
+}
+
+static PyType_Slot HeapCTypeMetaclassCustomNew_slots[] = {
+    { Py_tp_new, heap_ctype_metaclass_custom_tp_new },
+    {0},
+};
+
+static PyType_Spec HeapCTypeMetaclassCustomNew_spec = {
+    "_testcapi.HeapCTypeMetaclassCustomNew",
+    sizeof(PyHeapTypeObject),
+    sizeof(PyMemberDef),
+    Py_TPFLAGS_DEFAULT,
+    HeapCTypeMetaclassCustomNew_slots
+};
+
+
 typedef struct {
     PyObject_HEAD
     PyObject *dict;
@@ -7591,6 +7650,20 @@ PyInit__testcapi(void)
     Py_DECREF(subclass_with_finalizer_bases);
     PyModule_AddObject(m, "HeapCTypeSubclassWithFinalizer", HeapCTypeSubclassWithFinalizer);
 
+    PyObject *HeapCTypeMetaclass = PyType_FromMetaclass(
+        &PyType_Type, m, &HeapCTypeMetaclass_spec, (PyObject *) &PyType_Type);
+    if (HeapCTypeMetaclass == NULL) {
+        return NULL;
+    }
+    PyModule_AddObject(m, "HeapCTypeMetaclass", HeapCTypeMetaclass);
+
+    PyObject *HeapCTypeMetaclassCustomNew = PyType_FromMetaclass(
+        &PyType_Type, m, &HeapCTypeMetaclassCustomNew_spec, (PyObject *) &PyType_Type);
+    if (HeapCTypeMetaclassCustomNew == NULL) {
+        return NULL;
+    }
+    PyModule_AddObject(m, "HeapCTypeMetaclassCustomNew", HeapCTypeMetaclassCustomNew);
+
     if (PyType_Ready(&ContainerNoGC_type) < 0) {
         return NULL;
     }

Misc/stable_abi.toml

@@ -3366,13 +3366,8 @@ static const PySlot_Offset pyslot_offsets[] = {
 };
 
 PyObject *
-PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
-{
-    return PyType_FromModuleAndSpec(NULL, spec, bases);
-}
-
-PyObject *
-PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
+PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module,
+                     PyType_Spec *spec, PyObject *bases)
 {
     PyHeapTypeObject *res;
     PyObject *modname;
@@ -3384,6 +3379,16 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
     char *res_start;
     short slot_offset, subslot_offset;
 
+    if (!metaclass) {
+        metaclass = &PyType_Type;
+    }
+
+    if (metaclass->tp_new != PyType_Type.tp_new) {
+        PyErr_SetString(PyExc_TypeError,
+                        "Metaclasses with custom tp_new are not supported.");
+        return NULL;
+    }
+
     nmembers = weaklistoffset = dictoffset = vectorcalloffset = 0;
     for (slot = spec->slots; slot->slot; slot++) {
         if (slot->slot == Py_tp_members) {
@@ -3412,7 +3417,7 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
         }
     }
 
-    res = (PyHeapTypeObject*)PyType_GenericAlloc(&PyType_Type, nmembers);
+    res = (PyHeapTypeObject*)metaclass->tp_alloc(metaclass, nmembers);
     if (res == NULL)
         return NULL;
     res_start = (char*)res;
@@ -3639,10 +3644,22 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
     return NULL;
 }
 
+PyObject *
+PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
+{
+    return PyType_FromMetaclass(NULL, module, spec, bases);
+}
+
+PyObject *
+PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
+{
+    return PyType_FromMetaclass(NULL, NULL, spec, bases);
+}
+
 PyObject *
 PyType_FromSpec(PyType_Spec *spec)
 {
-    return PyType_FromSpecWithBases(spec, NULL);
+    return PyType_FromMetaclass(NULL, NULL, spec, NULL);
 }
 
 PyObject *