From 149ea9dc439c6b34fe07a14d74acbd60a70be243 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 23 Mar 2023 10:03:20 -0700 Subject: [PATCH] Deferred reference counting Initial partial implementation of deferred reference counting. --- Include/internal/pycore_object.h | 28 ++++++++++++++ Include/object.h | 24 +++++++++--- Modules/gcmodule.c | 65 +++++++++++++++++++++----------- Objects/descrobject.c | 1 + Objects/dictobject.c | 4 ++ Objects/funcobject.c | 7 +++- Objects/moduleobject.c | 3 ++ Objects/typeobject.c | 2 + Python/sysmodule.c | 5 ++- 9 files changed, 110 insertions(+), 29 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index e7acbb33796..abc8446b82f 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -360,6 +360,34 @@ _PyObject_SetMaybeWeakref(PyObject *op) } } +/* Marks the object as support deferred reference counting. + * + * The object's type must be GC-enabled. This function is not thread-safe with + * respect to concurrent modifications; it must be called before the object + * becomes visible to other threads. + * + * Deferred refcounted objects are marked as "queued" to prevent merging + * reference count fields outside the garbage collector. + */ +static inline void +_PyObject_SetDeferredRefcount(PyObject *op) +{ + assert(_Py_ThreadLocal(op) && "non thread-safe"); + assert(!_PyObject_HasDeferredRefcount(op) && "already uses deferred refcounting"); + assert(PyType_IS_GC(Py_TYPE(op))); + op->ob_ref_local += _Py_REF_DEFERRED_MASK + 1; + op->ob_ref_shared = (op->ob_ref_shared & ~_Py_REF_SHARED_FLAG_MASK) | _Py_REF_QUEUED; +} + +#define _PyObject_SET_DEFERRED_REFCOUNT(op) _PyObject_SetDeferredRefcount(_PyObject_CAST(op)) + +// Check is refcount is deferred or immortal +static inline int +_Py_REF_NON_IMMEDIATE(uint32_t local) +{ + return _Py_STATIC_CAST(int32_t, local) <= Py_REF_IMMORTAL; +} + #ifdef Py_REF_DEBUG extern void _PyDebug_PrintTotalRefs(void); #endif diff --git a/Include/object.h b/Include/object.h index b02ed8fb99e..82d0af04cd2 100644 --- a/Include/object.h +++ b/Include/object.h @@ -135,7 +135,7 @@ PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); #define Py_Is(x, y) ((x) == (y)) static inline void -_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal); +_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal, int *deferred); static inline void _PyRef_UnpackShared(uint32_t bits, Py_ssize_t *refcount, int *queued, int *merged); @@ -148,7 +148,7 @@ static inline Py_ssize_t Py_REFCNT(PyObject *ob) { int immortal; uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); - _PyRef_UnpackLocal(local, &local_refcount, &immortal); + _PyRef_UnpackLocal(local, &local_refcount, &immortal, NULL); if (immortal) { return 999; @@ -617,7 +617,7 @@ _Py_ThreadLocal(PyObject *op) #define _Py_REF_LOCAL_SHIFT 0 #define _Py_REF_LOCAL_INIT 1 #define Py_REF_IMMORTAL -1 -#define _Py_REF_DEFERRED_MASK 0x40000000 +#define _Py_REF_DEFERRED_MASK 0xC0000000 // 30 2 // [refcount] [bits] @@ -884,10 +884,17 @@ static inline PyObject* _Py_XNewRef(PyObject *obj) #endif static inline void -_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal) +_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal, int *deferred) { *refcount = bits >> _Py_REF_LOCAL_SHIFT; *immortal = _Py_REF_IS_IMMORTAL(bits); + int is_deferred = _Py_STATIC_CAST(int32_t, bits) < Py_REF_IMMORTAL; + if (is_deferred) { + *refcount -= _Py_REF_DEFERRED_MASK; + } + if (deferred) { + *deferred = is_deferred; + } } static inline void @@ -925,7 +932,7 @@ _PyObject_IsReferened(PyObject *op) int immortal; uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); - _PyRef_UnpackLocal(local, &local_refcount, &immortal); + _PyRef_UnpackLocal(local, &local_refcount, &immortal, NULL); if (immortal) { return 1; @@ -940,6 +947,13 @@ _PyObject_IsReferened(PyObject *op) return (local_refcount + shared_refcount) > 0; } +static inline int +_PyObject_HasDeferredRefcount(PyObject *op) +{ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + return _Py_STATIC_CAST(int32_t, local) < Py_REF_IMMORTAL; +} + static inline void _Py_RESURRECT(PyObject *op, Py_ssize_t refcnt) { diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 4c7934115d6..2977ffc7d2d 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -318,13 +318,13 @@ static Py_ssize_t _Py_GC_REFCNT(PyObject *op) { Py_ssize_t local, shared; - int immortal; + int immortal, deferred; - _PyRef_UnpackLocal(op->ob_ref_local, &local, &immortal); + _PyRef_UnpackLocal(op->ob_ref_local, &local, &immortal, &deferred); _PyRef_UnpackShared(op->ob_ref_shared, &shared, NULL, NULL); assert(!immortal); - return local + shared; + return local + shared - deferred; } typedef int (gc_visit_fn)(PyGC_Head* gc, void *arg); @@ -682,9 +682,35 @@ find_dead_shared_keys(_PyObjectQueue **queue, int *num_unmarked) } } +static void +merge_refcount(PyObject *op, Py_ssize_t extra) +{ + Py_ssize_t local_refcount, shared_refcount; + int immortal, deferred; + + assert(_PyRuntime.stop_the_world); + + _PyRef_UnpackLocal(op->ob_ref_local, &local_refcount, &immortal, &deferred); + _PyRef_UnpackShared(op->ob_ref_shared, &shared_refcount, NULL, NULL); + assert(!immortal && "immortal objects should not be in garbage"); + + Py_ssize_t refcount = local_refcount + shared_refcount; + refcount += extra; + refcount -= deferred; + +#ifdef Py_REF_DEBUG + _Py_IncRefTotalN(extra); +#endif + + op->ob_tid = 0; + op->ob_ref_local = 0; + op->ob_ref_shared = _Py_REF_PACK_SHARED(refcount, _Py_REF_MERGED); +} + struct update_refs_args { PyGC_Head *list; int split_keys_marked; + _PyGC_Reason gc_reason; }; // Compute the number of external references to objects in the heap @@ -730,6 +756,15 @@ update_refs(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size } } + if (arg->gc_reason == GC_REASON_SHUTDOWN) { + if (_PyObject_HasDeferredRefcount(op)) { + // Disable deferred reference counting when we're shutting down. + // This is useful for interp->sysdict because the last reference + // to it is cleared after the last GC cycle. + merge_refcount(op, 0); + } + } + // Add the actual refcount to gc_refs. Py_ssize_t refcount = _Py_GC_REFCNT(op); _PyObject_ASSERT(op, refcount >= 0); @@ -1008,23 +1043,7 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers) static void incref_merge(PyObject *op) { - Py_ssize_t local_refcount, shared_refcount; - int immortal; - - assert(_PyRuntime.stop_the_world); - -#ifdef Py_REF_DEBUG - _Py_IncRefTotal(); -#endif - - _PyRef_UnpackLocal(op->ob_ref_local, &local_refcount, &immortal); - _PyRef_UnpackShared(op->ob_ref_shared, &shared_refcount, NULL, NULL); - assert(!immortal && "immortal objects should not be in garbage"); - - Py_ssize_t refcount = local_refcount + shared_refcount + 1; - op->ob_tid = 0; - op->ob_ref_local = 0; - op->ob_ref_shared = _Py_REF_PACK_SHARED(refcount, _Py_REF_MERGED); + merge_refcount(op, 1); } /* Subtracts one from the refcount field. */ @@ -1594,7 +1613,11 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) validate_tracked_heap(_PyGC_PREV_MASK|_PyGC_PREV_MASK_UNREACHABLE, 0); gc_list_init(&young); - struct update_refs_args args = { .list = &young, .split_keys_marked = 0 }; + struct update_refs_args args = { + .list = &young, + .split_keys_marked = 0, + .gc_reason = reason, + }; visit_heaps2(mi_heap_tag_gc, update_refs, &args); _PyObjectQueue *dead_keys = NULL; diff --git a/Objects/descrobject.c b/Objects/descrobject.c index c545b90c628..0acedf6c906 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -903,6 +903,7 @@ descr_new(PyTypeObject *descrtype, PyTypeObject *type, const char *name) descr = (PyDescrObject *)PyType_GenericAlloc(descrtype, 0); if (descr != NULL) { + _PyObject_SET_DEFERRED_REFCOUNT(descr); descr->d_type = (PyTypeObject*)Py_XNewRef(type); descr->d_name = PyUnicode_InternFromString(name); if (descr->d_name == NULL) { diff --git a/Objects/dictobject.c b/Objects/dictobject.c index e8030315880..11debee8884 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1371,6 +1371,10 @@ _PyDict_MaybeUntrack(PyObject *op) if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op)) return; + if (_PyObject_HasDeferredRefcount(op)) { + return; + } + mp = (PyDictObject *) op; numentries = mp->ma_keys->dk_nentries; if (_PyDict_HasSplitTable(mp)) { diff --git a/Objects/funcobject.c b/Objects/funcobject.c index d5cf5b9277b..07fcd50e3e3 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -77,7 +77,9 @@ PyFunction_ClearWatcher(int watcher_id) PyFunctionObject * _PyFunction_FromConstructor(PyFrameConstructor *constr) { - + // NOTE(sgross): functions created via FrameConstructor do *not* use + // deferred reference counting because they are typically not part of cycles + // nor accessed by multiple threads. PyFunctionObject *op = PyObject_GC_New(PyFunctionObject, &PyFunction_Type); if (op == NULL) { return NULL; @@ -172,6 +174,9 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_annotations = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = 0; + if ((code_obj->co_flags & CO_NESTED) == 0) { + _PyObject_SET_DEFERRED_REFCOUNT(op); + } _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return (PyObject *)op; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index c7227b7dd1b..7bfe227ca61 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -107,6 +107,9 @@ new_module_notrack(PyTypeObject *mt) m->md_name = NULL; m->md_dict = PyDict_New(); if (m->md_dict != NULL) { + PyObject_GC_Track(m->md_dict); + _PyObject_SET_DEFERRED_REFCOUNT(m->md_dict); + _PyObject_SET_DEFERRED_REFCOUNT(m); return m; } Py_DECREF(m); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 458e43f7467..fe8e9af4aa1 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2887,6 +2887,7 @@ type_new_alloc(type_new_ctx *ctx) if (type == NULL) { return NULL; } + _PyObject_SET_DEFERRED_REFCOUNT(type); PyHeapTypeObject *et = (PyHeapTypeObject *)type; // Initialize tp_flags. @@ -3775,6 +3776,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, if (res == NULL) { goto finally; } + _PyObject_SET_DEFERRED_REFCOUNT(res); res_start = (char*)res; type = &res->ht_type; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f3e3ec9b1b0..82c68cc3296 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1876,9 +1876,9 @@ sys_getfullrefcount(PyObject *module, PyObject *object) uintptr_t tid = _PyObject_ThreadId(object); Py_ssize_t local, shared; - int immortal, queued, merged; + int immortal, deferred, queued, merged; - _PyRef_UnpackLocal(object->ob_ref_local, &local, &immortal); + _PyRef_UnpackLocal(object->ob_ref_local, &local, &immortal, &deferred); _PyRef_UnpackShared(object->ob_ref_shared, &shared, &queued, &merged); PyDict_SetItemString(res, "local", PyLong_FromSsize_t(local)); @@ -1886,6 +1886,7 @@ sys_getfullrefcount(PyObject *module, PyObject *object) PyDict_SetItemString(res, "merged", PyBool_FromLong(merged)); PyDict_SetItemString(res, "queued", PyBool_FromLong(queued)); PyDict_SetItemString(res, "immortal", PyBool_FromLong(immortal)); + PyDict_SetItemString(res, "deferred", PyBool_FromLong(deferred)); if (queued) { Py_INCREF(Py_None); PyDict_SetItemString(res, "tid", Py_None);