bpo-43688: Support the limited C API in debug mode (GH-25131)

The limited C API is now supported if Python is built in debug mode
(if the Py_DEBUG macro is defined). In the limited C API, the
Py_INCREF() and Py_DECREF() functions are now implemented as opaque
function calls, rather than accessing directly the PyObject.ob_refcnt
member, if Python is built in debug mode and the Py_LIMITED_API macro
targets Python 3.10 or newer. It became possible to support the
limited C API in debug mode because the PyObject structure is the
same in release and debug mode since Python 3.8 (see bpo-36465).

The limited C API is still not supported in the --with-trace-refs
special build (Py_TRACE_REFS macro).

Author: vstinner

Doc/whatsnew/3.10.rst

@@ -1315,6 +1315,18 @@ New Features
   to simulate.
   (Contributed by Antoine Pitrou in :issue:`43356`.)
 
+* The limited C API is now supported if Python is built in debug mode (if the
+  ``Py_DEBUG`` macro is defined). In the limited C API, the :c:func:`Py_INCREF`
+  and :c:func:`Py_DECREF` functions are now implemented as opaque function
+  calls, rather than accessing directly the :c:member:`PyObject.ob_refcnt`
+  member, if Python is built in debug mode and the ``Py_LIMITED_API`` macro
+  targets Python 3.10 or newer. It became possible to support the limited C API
+  in debug mode because the :c:type:`PyObject` structure is the same in release
+  and debug mode since Python 3.8 (see :issue:`36465`).
+
+  The limited C API is still not supported in the ``--with-trace-refs`` special
+  build (``Py_TRACE_REFS`` macro).
+  (Contributed by Victor Stinner in :issue:`43688`.)
 
 Porting to Python 3.10
 ----------------------

Include/object.h

@@ -54,11 +54,11 @@ whose size is determined when the object is allocated.
 
 /* Py_DEBUG implies Py_REF_DEBUG. */
 #if defined(Py_DEBUG) && !defined(Py_REF_DEBUG)
-#define Py_REF_DEBUG
+#  define Py_REF_DEBUG
 #endif
 
-#if defined(Py_LIMITED_API) && defined(Py_REF_DEBUG)
-#error Py_LIMITED_API is incompatible with Py_DEBUG, Py_TRACE_REFS, and Py_REF_DEBUG
+#if defined(Py_LIMITED_API) && defined(Py_TRACE_REFS)
+#  error Py_LIMITED_API is incompatible with Py_TRACE_REFS
 #endif
 
 /* PyTypeObject structure is defined in cpython/object.h.
@@ -74,8 +74,8 @@ typedef struct _typeobject PyTypeObject;
 #define _PyObject_EXTRA_INIT 0, 0,
 
 #else
-#define _PyObject_HEAD_EXTRA
-#define _PyObject_EXTRA_INIT
+#  define _PyObject_HEAD_EXTRA
+#  define _PyObject_EXTRA_INIT
 #endif
 
 /* PyObject_HEAD defines the initial segment of every PyObject. */
@@ -427,21 +427,46 @@ PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno,
 
 PyAPI_FUNC(void) _Py_Dealloc(PyObject *);
 
+/*
+These are provided as conveniences to Python runtime embedders, so that
+they can have object code that is not dependent on Python compilation flags.
+*/
+PyAPI_FUNC(void) Py_IncRef(PyObject *);
+PyAPI_FUNC(void) Py_DecRef(PyObject *);
+
+// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL.
+// Private functions used by Py_INCREF() and Py_DECREF().
+PyAPI_FUNC(void) _Py_IncRef(PyObject *);
+PyAPI_FUNC(void) _Py_DecRef(PyObject *);
+
 static inline void _Py_INCREF(PyObject *op)
 {
+#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
+    // Stable ABI for Python 3.10 built in debug mode.
+    _Py_IncRef(op);
+#else
+    // Non-limited C API and limited C API for Python 3.9 and older access
+    // directly PyObject.ob_refcnt.
 #ifdef Py_REF_DEBUG
     _Py_RefTotal++;
 #endif
     op->ob_refcnt++;
+#endif
 }
 #define Py_INCREF(op) _Py_INCREF(_PyObject_CAST(op))
 
 static inline void _Py_DECREF(
-#ifdef Py_REF_DEBUG
+#if defined(Py_REF_DEBUG) && !(defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000)
     const char *filename, int lineno,
 #endif
     PyObject *op)
 {
+#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
+    // Stable ABI for Python 3.10 built in debug mode.
+    _Py_DecRef(op);
+#else
+    // Non-limited C API and limited C API for Python 3.9 and older access
+    // directly PyObject.ob_refcnt.
 #ifdef Py_REF_DEBUG
     _Py_RefTotal--;
 #endif
@@ -455,8 +480,9 @@ static inline void _Py_DECREF(
     else {
         _Py_Dealloc(op);
     }
+#endif
 }
-#ifdef Py_REF_DEBUG
+#if defined(Py_REF_DEBUG) && !(defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000)
 #  define Py_DECREF(op) _Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))
 #else
 #  define Py_DECREF(op) _Py_DECREF(_PyObject_CAST(op))
@@ -525,13 +551,6 @@ static inline void _Py_XDECREF(PyObject *op)
 
 #define Py_XDECREF(op) _Py_XDECREF(_PyObject_CAST(op))
 
-/*
-These are provided as conveniences to Python runtime embedders, so that
-they can have object code that is not dependent on Python compilation flags.
-*/
-PyAPI_FUNC(void) Py_IncRef(PyObject *);
-PyAPI_FUNC(void) Py_DecRef(PyObject *);
-
 // Create a new strong reference to an object:
 // increment the reference count of the object and return the object.
 PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj);

Misc/NEWS.d/next/C API/2021-04-01-09-10-42.bpo-43688.G4gs6k.rst

@@ -0,0 +1,13 @@
+The limited C API is now supported if Python is built in debug mode (if the
+``Py_DEBUG`` macro is defined). In the limited C API, the :c:func:`Py_INCREF`
+and :c:func:`Py_DECREF` functions are now implemented as opaque function calls,
+rather than accessing directly the :c:member:`PyObject.ob_refcnt` member, if
+Python is built in debug mode and the ``Py_LIMITED_API`` macro targets Python
+3.10 or newer. It became possible to support the limited C API in debug mode
+because the :c:type:`PyObject` structure is the same in release and debug mode
+since Python 3.8 (see :issue:`36465`).
+
+The limited C API is still not supported in the ``--with-trace-refs`` special
+build (``Py_TRACE_REFS`` macro).
+
+Patch by Victor Stinner.

Objects/object.c

@@ -13,6 +13,11 @@
 #include "frameobject.h"
 #include "interpreteridobject.h"
 
+#ifdef Py_LIMITED_API
+   // Prevent recursive call _Py_IncRef() <=> Py_INCREF()
+#  error "Py_LIMITED_API macro must not be defined"
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -136,6 +141,18 @@ Py_DecRef(PyObject *o)
     Py_XDECREF(o);
 }
 
+void
+_Py_IncRef(PyObject *o)
+{
+    Py_INCREF(o);
+}
+
+void
+_Py_DecRef(PyObject *o)
+{
+    Py_DECREF(o);
+}
+
 PyObject *
 PyObject_Init(PyObject *op, PyTypeObject *tp)
 {

PC/python3dll.c

@@ -36,6 +36,8 @@ EXPORT_FUNC(_PyTrash_deposit_object)
 EXPORT_FUNC(_PyTrash_destroy_chain)
 EXPORT_FUNC(_PyTrash_thread_deposit_object)
 EXPORT_FUNC(_PyTrash_thread_destroy_chain)
+EXPORT_FUNC(_Py_IncRef)
+EXPORT_FUNC(_Py_DecRef)
 EXPORT_FUNC(Py_AddPendingCall)
 EXPORT_FUNC(Py_AtExit)
 EXPORT_FUNC(Py_BuildValue)

setup.py

@@ -1864,17 +1864,11 @@ def detect_modules(self):
 ##         # Uncomment these lines if you want to play with xxmodule.c
 ##         self.add(Extension('xx', ['xxmodule.c']))
 
-        if 'd' not in sysconfig.get_config_var('ABIFLAGS'):
-            # Non-debug mode: Build xxlimited with limited API
-            self.add(Extension('xxlimited', ['xxlimited.c'],
-                               define_macros=[('Py_LIMITED_API', '0x030a0000')]))
-            self.add(Extension('xxlimited_35', ['xxlimited_35.c'],
-                               define_macros=[('Py_LIMITED_API', '0x03050000')]))
-        else:
-            # Debug mode: Build xxlimited with the full API
-            # (which is compatible with the limited one)
-            self.add(Extension('xxlimited', ['xxlimited.c']))
-            self.add(Extension('xxlimited_35', ['xxlimited_35.c']))
+        # Limited C API
+        self.add(Extension('xxlimited', ['xxlimited.c'],
+                           define_macros=[('Py_LIMITED_API', '0x030a0000')]))
+        self.add(Extension('xxlimited_35', ['xxlimited_35.c'],
+                           define_macros=[('Py_LIMITED_API', '0x03050000')]))
 
     def detect_tkinter_fromenv(self):
         # Build _tkinter using the Tcl/Tk locations specified by