pythongh-112529: Use GC mimalloc heaps for GC objects

Include/internal/pycore_object_alloc.h

@@ -0,0 +1,64 @@
+#ifndef Py_INTERNAL_OBJECT_ALLOC_H
+#define Py_INTERNAL_OBJECT_ALLOC_H
+
+#include "pycore_object.h" // _PyType_HasFeature()
+#include "pycore_pystate.h" // _PyThreadState_GET()
+#include "pycore_tstate.h" // _PyThreadStateImpl
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+// Sets the heap used for PyObject_Malloc(), PyObject_Realloc(), etc. calls in
+// Py_GIL_DISABLED builds. We use different heaps depending on if the object
+// supports GC and if it has a pre-header. We smuggle the choice of heap
+// through the _mimalloc_thread_state.
+// This is a no-op outside of Py_GIL_DISABLED builds.
+static inline void *
+_PyObject_MallocWithType(PyTypeObject *tp, size_t size)
+{
+#ifdef Py_GIL_DISABLED
+ _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+ struct _mimalloc_thread_state *m = &tstate->mimalloc;
+ if (_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER)) {
+ m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_GC_PRE];
+ }
+ else if (_PyType_IS_GC(tp)) {
+ m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_GC];
+ }
+#endif
+ void *mem = PyObject_Malloc(size);
+#ifdef Py_GIL_DISABLED
+ m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
+#endif
+ return mem;
+}
+
+static inline void *
+_PyObject_ReallocWithType(PyTypeObject *tp, void *ptr, size_t size)
+{
+#ifdef Py_GIL_DISABLED
+ _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+ struct _mimalloc_thread_state *m = &tstate->mimalloc;
+ if (_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER)) {
+ m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_GC_PRE];
+ }
+ else if (_PyType_IS_GC(tp)) {
+ m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_GC];
+ }
+#endif
+ void *mem = PyObject_Realloc(ptr, size);
+#ifdef Py_GIL_DISABLED
+ m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
+#endif
+ return mem;
+}
+
+#ifdef __cplusplus
+}
+#endif
+#endif // !Py_INTERNAL_OBJECT_ALLOC_H

Objects/typeobject.c

@@ -11,6 +11,7 @@
 #include "pycore_modsupport.h" // _PyArg_NoKwnames()
 #include "pycore_moduleobject.h" // _PyModule_GetDef()
 #include "pycore_object.h" // _PyType_HasFeature()
+#include "pycore_object_alloc.h" // _PyObject_MallocWithType()
 #include "pycore_pyerrors.h" // _PyErr_Occurred()
 #include "pycore_pystate.h" // _PyThreadState_GET()
 #include "pycore_symtable.h" // _Py_Mangle()
@@ -1729,7 +1730,7 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
  const size_t size = _PyObject_VAR_SIZE(type, nitems+1);
 
  const size_t presize = _PyType_PreHeaderSize(type);
- char *alloc = PyObject_Malloc(size + presize);
+ char *alloc = _PyObject_MallocWithType(type, size + presize);
  if (alloc == NULL) {
  return PyErr_NoMemory();
  }