From ffe7d7be96bbd7e9b3e20844cfa27d9cacbbde5b Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Sun, 23 Jan 2022 03:44:57 -0500 Subject: [PATCH 01/14] Add specializations of Py_DECREF when types are known --- Include/boolobject.h | 16 ++++++ Include/internal/pycore_floatobject.h | 10 ++++ Include/internal/pycore_long.h | 10 ++++ Include/internal/pycore_unicodeobject.h | 8 +++ Include/object.h | 33 ++++++++++++ Modules/mathmodule.c | 11 ++-- Objects/floatobject.c | 35 +++++++----- Objects/longobject.c | 33 +++++++----- Objects/unicodeobject.c | 7 +++ Python/bltinmodule.c | 2 +- Python/ceval.c | 72 ++++++++++++------------- 11 files changed, 170 insertions(+), 67 deletions(-) diff --git a/Include/boolobject.h b/Include/boolobject.h index cda6f89a99e9a2..671cf5c5b95f74 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -30,6 +30,22 @@ PyAPI_FUNC(int) Py_IsTrue(PyObject *x); PyAPI_FUNC(int) Py_IsFalse(PyObject *x); #define Py_IsFalse(x) Py_Is((x), Py_False) +static inline void +_Py_DECREF_BOOL(PyObject *op) +{ + assert(PyBool_Check(op)); +#ifdef Py_REF_DEBUG + _Py_RefTotal--; +#endif + op->ob_refcnt--; +#ifdef Py_DEBUG + if (op->ob_refcnt <= 0) { + // Calls _Py_FatalRefcountError + _Py_Dealloc(op); + } +#endif +} + /* Macros for returning Py_True or Py_False, respectively */ #define Py_RETURN_TRUE return Py_NewRef(Py_True) #define Py_RETURN_FALSE return Py_NewRef(Py_False) diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 891e422f594721..5e92c3c3e0c58f 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -38,6 +38,16 @@ struct _Py_float_state { #endif }; +PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyFloatObject *op); + +static inline void +_Py_DECREF_FLOAT(PyObject *op) +{ + assert(PyFloat_CheckExact(op)); + _Py_DECREF_SPECIALIZED(op, (destructor)_PyFloat_ExactDealloc); +} + + /* _PyFloat_{Pack,Unpack}{4,8} * * The struct and pickle (at least) modules need an efficient platform- diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 436bf084685997..2139f6fa523652 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -67,6 +67,16 @@ PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( int base, int alternate); +PyAPI_FUNC(void) _PyLong_ExactDealloc(PyLongObject *op); + +static inline void +_Py_DECREF_INT(PyObject *op) +{ + assert(PyLong_CheckExact(op)); + _Py_DECREF_SPECIALIZED(op, (destructor)_PyLong_ExactDealloc); +} + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index fabe522f6fc23b..bba6a8c626162e 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -10,6 +10,14 @@ extern "C" { #include "pycore_fileutils.h" // _Py_error_handler +void _PyUnicode_ExactDealloc(PyUnicodeObject *op); + +static inline void +_Py_DECREF_STR(PyObject *op) +{ + assert(PyUnicode_CheckExact(op)); + _Py_DECREF_SPECIALIZED(op, (destructor)_PyUnicode_ExactDealloc); +} /* runtime lifecycle */ diff --git a/Include/object.h b/Include/object.h index 4fd16616ce705e..f8f0c3a3172c16 100644 --- a/Include/object.h +++ b/Include/object.h @@ -524,6 +524,22 @@ static inline void _Py_DECREF( # define Py_DECREF(op) _Py_DECREF(_PyObject_CAST(op)) #endif +static inline void +_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) +{ +#ifdef Py_REF_DEBUG + _Py_RefTotal--; +#endif + if (--op->ob_refcnt != 0) { + assert(op->ob_refcnt > 0); + } + else { +#ifdef Py_TRACE_REFS + _Py_ForgetReference(op); +#endif + destruct(op); + } +} /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear * and tp_dealloc implementations. @@ -622,6 +638,23 @@ Don't forget to apply Py_INCREF() when returning this value!!! PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ #define Py_None (&_Py_NoneStruct) +static inline void +_Py_DECREF_NONE(PyObject *op) +{ + assert(op == Py_None); +#ifdef Py_REF_DEBUG + _Py_RefTotal--; +#endif + (void)op; + ((PyObject *)Py_None)->ob_refcnt--; +#ifdef Py_DEBUG + if (((PyObject *)Py_None)->ob_refcnt <= 0) { + // Calls _Py_FatalRefcountError + _Py_Dealloc(op); + } +#endif +} + // Test if an object is the None singleton, the same as "x is None" in Python. PyAPI_FUNC(int) Py_IsNone(PyObject *x); #define Py_IsNone(x) Py_Is((x), Py_None) diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 0c7d4de0686213..8038bb3a5d6947 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -60,7 +60,8 @@ raised for division by zero and mod by zero. #include "pycore_bitutils.h" // _Py_bit_length() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_dtoa.h" // _Py_dg_infinity() -#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_long.h" // _PyLong_GetZero(), _Py_DECREF_INT +#include "pycore_floatobject.h" // _Py_DECREF_FLOAT /* For DBL_EPSILON in _math.h */ #include /* For _Py_log1p with workarounds for buggy handling of zeros. */ @@ -3139,7 +3140,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) long i_result = PyLong_AsLongAndOverflow(result, &overflow); /* If this already overflowed, don't even enter the loop. */ if (overflow == 0) { - Py_DECREF(result); + _Py_DECREF_INT(result); result = NULL; } /* Loop over all the items in the iterable until we finish, we overflow @@ -3158,7 +3159,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) if (overflow == 0 && !_check_long_mult_overflow(i_result, b)) { long x = i_result * b; i_result = x; - Py_DECREF(item); + _Py_DECREF_INT(item); continue; } } @@ -3187,7 +3188,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) */ if (PyFloat_CheckExact(result)) { double f_result = PyFloat_AS_DOUBLE(result); - Py_DECREF(result); + _Py_DECREF_FLOAT(result); result = NULL; while(result == NULL) { item = PyIter_Next(iter); @@ -3200,7 +3201,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) } if (PyFloat_CheckExact(item)) { f_result *= PyFloat_AS_DOUBLE(item); - Py_DECREF(item); + _Py_DECREF_FLOAT(item); continue; } if (PyLong_CheckExact(item)) { diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 79fbdabce96081..6e5bb3d455e058 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -238,23 +238,34 @@ PyFloat_FromString(PyObject *v) return result; } +void +_PyFloat_ExactDealloc(PyFloatObject *op) +{ + assert(PyFloat_CheckExact(op)); +#if PyFloat_MAXFREELIST > 0 + struct _Py_float_state *state = get_float_state(); +#ifdef Py_DEBUG + // float_dealloc() must not be called after _PyFloat_Fini() + assert(state->numfree != -1); +#endif + if (state->numfree >= PyFloat_MAXFREELIST) { + PyObject_Free(op); + return; + } + state->numfree++; + Py_SET_TYPE(op, (PyTypeObject *)state->free_list); + state->free_list = op; +#else + PyObject_Free(op); +#endif +} + static void float_dealloc(PyFloatObject *op) { #if PyFloat_MAXFREELIST > 0 if (PyFloat_CheckExact(op)) { - struct _Py_float_state *state = get_float_state(); -#ifdef Py_DEBUG - // float_dealloc() must not be called after _PyFloat_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree >= PyFloat_MAXFREELIST) { - PyObject_Free(op); - return; - } - state->numfree++; - Py_SET_TYPE(op, (PyTypeObject *)state->free_list); - state->free_list = op; + _PyFloat_ExactDealloc(op); } else #endif diff --git a/Objects/longobject.c b/Objects/longobject.c index 7721f40adbba65..5e58cd47808417 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -61,7 +61,7 @@ maybe_small_long(PyLongObject *v) if (v && IS_MEDIUM_VALUE(v)) { stwodigits ival = medium_value(v); if (IS_SMALL_INT(ival)) { - Py_DECREF(v); + _Py_DECREF_INT((PyObject *)v); return (PyLongObject *)get_small_int((sdigit)ival); } } @@ -1824,7 +1824,7 @@ long_to_decimal_string_internal(PyObject *aa, #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS - Py_DECREF(scratch); + _Py_DECREF_INT((PyObject *)scratch); if (writer) { writer->pos += strlen; } @@ -3479,15 +3479,15 @@ k_mul(PyLongObject *a, PyLongObject *b) */ i = Py_SIZE(ret) - shift; /* # digits after shift */ (void)v_isub(ret->ob_digit + shift, i, t2->ob_digit, Py_SIZE(t2)); - Py_DECREF(t2); + _Py_DECREF_INT((PyObject *)t2); (void)v_isub(ret->ob_digit + shift, i, t1->ob_digit, Py_SIZE(t1)); - Py_DECREF(t1); + _Py_DECREF_INT((PyObject *)t1); /* 6. t3 <- (ah+al)(bh+bl), and add into result. */ if ((t1 = x_add(ah, al)) == NULL) goto fail; - Py_DECREF(ah); - Py_DECREF(al); + _Py_DECREF_INT((PyObject *)ah); + _Py_DECREF_INT((PyObject *)al); ah = al = NULL; if (a == b) { @@ -3498,13 +3498,13 @@ k_mul(PyLongObject *a, PyLongObject *b) Py_DECREF(t1); goto fail; } - Py_DECREF(bh); - Py_DECREF(bl); + _Py_DECREF_INT((PyObject *)bh); + _Py_DECREF_INT((PyObject *)bl); bh = bl = NULL; t3 = k_mul(t1, t2); - Py_DECREF(t1); - Py_DECREF(t2); + _Py_DECREF_INT((PyObject *)t1); + _Py_DECREF_INT((PyObject *)t2); if (t3 == NULL) goto fail; assert(Py_SIZE(t3) >= 0); @@ -3512,7 +3512,7 @@ k_mul(PyLongObject *a, PyLongObject *b) * See the (*) comment after this function. */ (void)v_iadd(ret->ob_digit + shift, i, t3->ob_digit, Py_SIZE(t3)); - Py_DECREF(t3); + _Py_DECREF_INT((PyObject *)t3); return long_normalize(ret); @@ -3617,13 +3617,13 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b) /* Add into result. */ (void)v_iadd(ret->ob_digit + nbdone, Py_SIZE(ret) - nbdone, product->ob_digit, Py_SIZE(product)); - Py_DECREF(product); + _Py_DECREF_INT((PyObject *)product); bsize -= nbtouse; nbdone += nbtouse; } - Py_DECREF(bslice); + _Py_DECREF_INT((PyObject *)bslice); return long_normalize(ret); fail: @@ -5743,6 +5743,13 @@ int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, return long_obj; } +void +_PyLong_ExactDealloc(PyLongObject *op) +{ + assert(PyLong_CheckExact(op)); + PyObject_Del(op); +} + static PyObject * long_long_meth(PyObject *self, PyObject *Py_UNUSED(ignored)) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4cea0d8e62e858..6698d73394744d 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15463,6 +15463,13 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) return NULL; } +void +_PyUnicode_ExactDealloc(PyUnicodeObject *op) +{ + assert(PyUnicode_CheckExact(op)); + PyObject_Free(op); +} + PyDoc_STRVAR(unicode_doc, "str(object='') -> str\n\ str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ed612091d83941..6dcb23694d2fcf 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2520,7 +2520,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) } if (PyFloat_CheckExact(item)) { f_result += PyFloat_AS_DOUBLE(item); - Py_DECREF(item); + _Py_DECREF_FLOAT(item); continue; } if (PyLong_Check(item)) { diff --git a/Python/ceval.c b/Python/ceval.c index 9aaddd99edacf7..6977b1cb4769d4 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2024,8 +2024,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(BINARY_OP, hit); PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(prod); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_INT(right); + _Py_DECREF_INT(left); STACK_SHRINK(1); if (prod == NULL) { goto error; @@ -2043,8 +2043,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr ((PyFloatObject *)right)->ob_fval; PyObject *prod = PyFloat_FromDouble(dprod); SET_SECOND(prod); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_FLOAT(right); + _Py_DECREF_FLOAT(left); STACK_SHRINK(1); if (prod == NULL) { goto error; @@ -2060,8 +2060,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(BINARY_OP, hit); PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sub); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_INT(right); + _Py_DECREF_INT(left); STACK_SHRINK(1); if (sub == NULL) { goto error; @@ -2078,8 +2078,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; PyObject *sub = PyFloat_FromDouble(dsub); SET_SECOND(sub); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_FLOAT(right); + _Py_DECREF_FLOAT(left); STACK_SHRINK(1); if (sub == NULL) { goto error; @@ -2096,8 +2096,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *res = PyUnicode_Concat(left, right); STACK_SHRINK(1); SET_TOP(res); - Py_DECREF(left); - Py_DECREF(right); + _Py_DECREF_STR(left); + _Py_DECREF_STR(right); if (TOP() == NULL) { goto error; } @@ -2122,10 +2122,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DEOPT_IF(var != left, BINARY_OP); STAT_INC(BINARY_OP, hit); GETLOCAL(next_oparg) = NULL; - Py_DECREF(left); + _Py_DECREF_STR(left); STACK_SHRINK(1); PyUnicode_Append(&TOP(), right); - Py_DECREF(right); + _Py_DECREF_STR(right); if (TOP() == NULL) { goto error; } @@ -2142,8 +2142,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr ((PyFloatObject *)right)->ob_fval; PyObject *sum = PyFloat_FromDouble(dsum); SET_SECOND(sum); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_FLOAT(right); + _Py_DECREF_FLOAT(left); STACK_SHRINK(1); if (sum == NULL) { goto error; @@ -2159,8 +2159,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(BINARY_OP, hit); PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sum); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_INT(right); + _Py_DECREF_INT(left); STACK_SHRINK(1); if (sum == NULL) { goto error; @@ -2218,7 +2218,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(res != NULL); Py_INCREF(res); STACK_SHRINK(1); - Py_DECREF(sub); + _Py_DECREF_INT(sub); SET_TOP(res); Py_DECREF(list); DISPATCH(); @@ -2241,7 +2241,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(res != NULL); Py_INCREF(res); STACK_SHRINK(1); - Py_DECREF(sub); + _Py_DECREF_INT(sub); SET_TOP(res); Py_DECREF(tuple); DISPATCH(); @@ -2372,7 +2372,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STACK_SHRINK(3); assert(old_value != NULL); Py_DECREF(old_value); - Py_DECREF(sub); + _Py_DECREF_INT(sub); Py_DECREF(list); DISPATCH(); } @@ -3211,12 +3211,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto error; } str = _PyUnicode_JoinArray(empty, stack_pointer - oparg, oparg); - Py_DECREF(empty); + _Py_DECREF_STR(empty); if (str == NULL) goto error; while (--oparg >= 0) { PyObject *item = POP(); - Py_DECREF(item); + _Py_DECREF_STR(item); } PUSH(str); DISPATCH(); @@ -3741,8 +3741,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(COMPARE_OP, hit); NEXTOPARG(); STACK_SHRINK(2); - Py_DECREF(left); - Py_DECREF(right); + _Py_DECREF_FLOAT(left); + _Py_DECREF_FLOAT(right); assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE); int jump = (1 << (sign + 1)) & when_to_jump_mask; if (!jump) { @@ -3774,8 +3774,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr int sign = (ileft > iright) - (ileft < iright); NEXTOPARG(); STACK_SHRINK(2); - Py_DECREF(left); - Py_DECREF(right); + _Py_DECREF_INT(left); + _Py_DECREF_INT(right); assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE); int jump = (1 << (sign + 1)) & when_to_jump_mask; if (!jump) { @@ -3808,8 +3808,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr NEXTOPARG(); assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE); STACK_SHRINK(2); - Py_DECREF(left); - Py_DECREF(right); + _Py_DECREF_STR(left); + _Py_DECREF_STR(right); assert(res == 0 || res == 1); assert(invert == 0 || invert == 1); int jump = res ^ invert; @@ -3878,7 +3878,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } if (Py_IsNone(match)) { - Py_DECREF(match); + _Py_DECREF_NONE(match); Py_XDECREF(rest); /* no match - jump to target */ JUMPTO(oparg); @@ -3981,11 +3981,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *cond = POP(); int err; if (Py_IsTrue(cond)) { - Py_DECREF(cond); + _Py_DECREF_BOOL(cond); DISPATCH(); } if (Py_IsFalse(cond)) { - Py_DECREF(cond); + _Py_DECREF_BOOL(cond); JUMPTO(oparg); CHECK_EVAL_BREAKER(); DISPATCH(); @@ -4008,11 +4008,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *cond = POP(); int err; if (Py_IsFalse(cond)) { - Py_DECREF(cond); + _Py_DECREF_BOOL(cond); DISPATCH(); } if (Py_IsTrue(cond)) { - Py_DECREF(cond); + _Py_DECREF_BOOL(cond); JUMPTO(oparg); CHECK_EVAL_BREAKER(); DISPATCH(); @@ -4038,14 +4038,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr CHECK_EVAL_BREAKER(); DISPATCH(); } - Py_DECREF(value); + _Py_DECREF_NONE(value); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = POP(); if (Py_IsNone(value)) { - Py_DECREF(value); + _Py_DECREF_NONE(value); JUMPTO(oparg); CHECK_EVAL_BREAKER(); DISPATCH(); @@ -4059,7 +4059,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr int err; if (Py_IsTrue(cond)) { STACK_SHRINK(1); - Py_DECREF(cond); + _Py_DECREF_BOOL(cond); DISPATCH(); } if (Py_IsFalse(cond)) { @@ -4083,7 +4083,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr int err; if (Py_IsFalse(cond)) { STACK_SHRINK(1); - Py_DECREF(cond); + _Py_DECREF_BOOL(cond); DISPATCH(); } if (Py_IsTrue(cond)) { From d2d7b6fe4861e80f473cd8632a78eee374ce9c6e Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 25 Jan 2022 05:39:40 +0000 Subject: [PATCH 02/14] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst new file mode 100644 index 00000000000000..f00011bae01516 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst @@ -0,0 +1 @@ +Add type-specialized versions of the ``Py_DECREF()`` macro for ``float``, ``int``, ``str``, ``bool``, and ``None`` to avoid pointer-chasing at runtime when types are known at compile time. \ No newline at end of file From 5cb77fd04d86c48b18986700a1069704becf5d2d Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Tue, 25 Jan 2022 03:05:51 -0500 Subject: [PATCH 03/14] Strings should be freed with unicode_dealloc, not PyObject_Free! --- Objects/unicodeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 6698d73394744d..659db01c38a84f 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15467,7 +15467,7 @@ void _PyUnicode_ExactDealloc(PyUnicodeObject *op) { assert(PyUnicode_CheckExact(op)); - PyObject_Free(op); + unicode_dealloc((PyObject *)op); } PyDoc_STRVAR(unicode_doc, From f58ebcfe7c0025f5991de26f1b33b4dd6700c111 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Tue, 25 Jan 2022 12:52:17 -0500 Subject: [PATCH 04/14] _Py_DECREF_IMMORTAL --- Include/boolobject.h | 16 ---------------- Include/object.h | 31 +++++++++++++++---------------- Python/ceval.c | 18 +++++++++--------- 3 files changed, 24 insertions(+), 41 deletions(-) diff --git a/Include/boolobject.h b/Include/boolobject.h index 671cf5c5b95f74..cda6f89a99e9a2 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -30,22 +30,6 @@ PyAPI_FUNC(int) Py_IsTrue(PyObject *x); PyAPI_FUNC(int) Py_IsFalse(PyObject *x); #define Py_IsFalse(x) Py_Is((x), Py_False) -static inline void -_Py_DECREF_BOOL(PyObject *op) -{ - assert(PyBool_Check(op)); -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif - op->ob_refcnt--; -#ifdef Py_DEBUG - if (op->ob_refcnt <= 0) { - // Calls _Py_FatalRefcountError - _Py_Dealloc(op); - } -#endif -} - /* Macros for returning Py_True or Py_False, respectively */ #define Py_RETURN_TRUE return Py_NewRef(Py_True) #define Py_RETURN_FALSE return Py_NewRef(Py_False) diff --git a/Include/object.h b/Include/object.h index f8f0c3a3172c16..50a0a1a8211639 100644 --- a/Include/object.h +++ b/Include/object.h @@ -541,6 +541,21 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) } } +static inline void +_Py_DECREF_IMMORTAL(PyObject *op) +{ +#ifdef Py_REF_DEBUG + _Py_RefTotal--; +#endif + op->ob_refcnt--; +#ifdef Py_DEBUG + if (op->ob_refcnt <= 0) { + // Calls _Py_FatalRefcountError for None, True, and False + _Py_Dealloc(op); + } +#endif +} + /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear * and tp_dealloc implementations. * @@ -638,22 +653,6 @@ Don't forget to apply Py_INCREF() when returning this value!!! PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ #define Py_None (&_Py_NoneStruct) -static inline void -_Py_DECREF_NONE(PyObject *op) -{ - assert(op == Py_None); -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif - (void)op; - ((PyObject *)Py_None)->ob_refcnt--; -#ifdef Py_DEBUG - if (((PyObject *)Py_None)->ob_refcnt <= 0) { - // Calls _Py_FatalRefcountError - _Py_Dealloc(op); - } -#endif -} // Test if an object is the None singleton, the same as "x is None" in Python. PyAPI_FUNC(int) Py_IsNone(PyObject *x); diff --git a/Python/ceval.c b/Python/ceval.c index 6977b1cb4769d4..f73829c170a63b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3878,7 +3878,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr } if (Py_IsNone(match)) { - _Py_DECREF_NONE(match); + _Py_DECREF_IMMORTAL(match); Py_XDECREF(rest); /* no match - jump to target */ JUMPTO(oparg); @@ -3981,11 +3981,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *cond = POP(); int err; if (Py_IsTrue(cond)) { - _Py_DECREF_BOOL(cond); + _Py_DECREF_IMMORTAL(cond); DISPATCH(); } if (Py_IsFalse(cond)) { - _Py_DECREF_BOOL(cond); + _Py_DECREF_IMMORTAL(cond); JUMPTO(oparg); CHECK_EVAL_BREAKER(); DISPATCH(); @@ -4008,11 +4008,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *cond = POP(); int err; if (Py_IsFalse(cond)) { - _Py_DECREF_BOOL(cond); + _Py_DECREF_IMMORTAL(cond); DISPATCH(); } if (Py_IsTrue(cond)) { - _Py_DECREF_BOOL(cond); + _Py_DECREF_IMMORTAL(cond); JUMPTO(oparg); CHECK_EVAL_BREAKER(); DISPATCH(); @@ -4038,14 +4038,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr CHECK_EVAL_BREAKER(); DISPATCH(); } - _Py_DECREF_NONE(value); + _Py_DECREF_IMMORTAL(value); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = POP(); if (Py_IsNone(value)) { - _Py_DECREF_NONE(value); + _Py_DECREF_IMMORTAL(value); JUMPTO(oparg); CHECK_EVAL_BREAKER(); DISPATCH(); @@ -4059,7 +4059,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr int err; if (Py_IsTrue(cond)) { STACK_SHRINK(1); - _Py_DECREF_BOOL(cond); + _Py_DECREF_IMMORTAL(cond); DISPATCH(); } if (Py_IsFalse(cond)) { @@ -4083,7 +4083,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr int err; if (Py_IsFalse(cond)) { STACK_SHRINK(1); - _Py_DECREF_BOOL(cond); + _Py_DECREF_IMMORTAL(cond); DISPATCH(); } if (Py_IsTrue(cond)) { From 0046cd7aeb4e21d99254148fe58d27180abe99e7 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Tue, 25 Jan 2022 13:32:11 -0500 Subject: [PATCH 05/14] Remove a layer of macros. Retain _Py_DECREF_INT for private use in longobject.c --- Include/internal/pycore_floatobject.h | 7 --- Include/internal/pycore_long.h | 8 --- Include/internal/pycore_unicodeobject.h | 7 --- .../2022-01-25-05-39-38.bpo-46509.ljrqrc.rst | 2 +- Modules/mathmodule.c | 14 ++--- Objects/longobject.c | 37 ++++++++----- Python/bltinmodule.c | 2 +- Python/ceval.c | 54 +++++++++---------- 8 files changed, 60 insertions(+), 71 deletions(-) diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 5e92c3c3e0c58f..e553d9ae72aed8 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -40,13 +40,6 @@ struct _Py_float_state { PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyFloatObject *op); -static inline void -_Py_DECREF_FLOAT(PyObject *op) -{ - assert(PyFloat_CheckExact(op)); - _Py_DECREF_SPECIALIZED(op, (destructor)_PyFloat_ExactDealloc); -} - /* _PyFloat_{Pack,Unpack}{4,8} * diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 2139f6fa523652..a0ed2f471d8b8f 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -69,14 +69,6 @@ PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( PyAPI_FUNC(void) _PyLong_ExactDealloc(PyLongObject *op); -static inline void -_Py_DECREF_INT(PyObject *op) -{ - assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED(op, (destructor)_PyLong_ExactDealloc); -} - - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index bba6a8c626162e..3eb5e359231a98 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -12,13 +12,6 @@ extern "C" { void _PyUnicode_ExactDealloc(PyUnicodeObject *op); -static inline void -_Py_DECREF_STR(PyObject *op) -{ - assert(PyUnicode_CheckExact(op)); - _Py_DECREF_SPECIALIZED(op, (destructor)_PyUnicode_ExactDealloc); -} - /* runtime lifecycle */ extern void _PyUnicode_InitState(PyInterpreterState *); diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst index f00011bae01516..08c2ccb976fe6e 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst @@ -1 +1 @@ -Add type-specialized versions of the ``Py_DECREF()`` macro for ``float``, ``int``, ``str``, ``bool``, and ``None`` to avoid pointer-chasing at runtime when types are known at compile time. \ No newline at end of file +Add type-specialized versions of the ``Py_DECREF()``, and use them for ``float``, ``int``, ``str``, ``bool``, and ``None`` to avoid pointer-chasing at runtime where types are known at C compile time. \ No newline at end of file diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 8038bb3a5d6947..058dbf698cabf0 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -60,8 +60,8 @@ raised for division by zero and mod by zero. #include "pycore_bitutils.h" // _Py_bit_length() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_dtoa.h" // _Py_dg_infinity() -#include "pycore_long.h" // _PyLong_GetZero(), _Py_DECREF_INT -#include "pycore_floatobject.h" // _Py_DECREF_FLOAT +#include "pycore_long.h" // _PyLong_GetZero(), _PyLong_ExactDealloc +#include "pycore_floatobject.h" // _PyFloat_ExactDealloc /* For DBL_EPSILON in _math.h */ #include /* For _Py_log1p with workarounds for buggy handling of zeros. */ @@ -3140,7 +3140,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) long i_result = PyLong_AsLongAndOverflow(result, &overflow); /* If this already overflowed, don't even enter the loop. */ if (overflow == 0) { - _Py_DECREF_INT(result); + _Py_DECREF_SPECIALIZED(result, (destructor)_PyLong_ExactDealloc); result = NULL; } /* Loop over all the items in the iterable until we finish, we overflow @@ -3159,7 +3159,8 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) if (overflow == 0 && !_check_long_mult_overflow(i_result, b)) { long x = i_result * b; i_result = x; - _Py_DECREF_INT(item); + _Py_DECREF_SPECIALIZED(item, + (destructor)_PyLong_ExactDealloc); continue; } } @@ -3188,7 +3189,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) */ if (PyFloat_CheckExact(result)) { double f_result = PyFloat_AS_DOUBLE(result); - _Py_DECREF_FLOAT(result); + _Py_DECREF_SPECIALIZED(result, (destructor)_PyFloat_ExactDealloc); result = NULL; while(result == NULL) { item = PyIter_Next(iter); @@ -3201,7 +3202,8 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) } if (PyFloat_CheckExact(item)) { f_result *= PyFloat_AS_DOUBLE(item); - _Py_DECREF_FLOAT(item); + _Py_DECREF_SPECIALIZED(item, + (destructor)_PyFloat_ExactDealloc); continue; } if (PyLong_CheckExact(item)) { diff --git a/Objects/longobject.c b/Objects/longobject.c index 5e58cd47808417..3be90da34ff239 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -39,7 +39,16 @@ medium_value(PyLongObject *x) #define IS_SMALL_INT(ival) (-_PY_NSMALLNEGINTS <= (ival) && (ival) < _PY_NSMALLPOSINTS) #define IS_SMALL_UINT(ival) ((ival) < _PY_NSMALLPOSINTS) -static inline int is_medium_int(stwodigits x) +static inline void +_Py_DECREF_INT(PyLongObject *op) +{ + assert(PyLong_CheckExact(op)); + _Py_DECREF_SPECIALIZED((PyObject *)op, + (destructor)_PyLong_ExactDealloc); +} + +static inline int +is_medium_int(stwodigits x) { /* Take care that we are comparing unsigned values. */ twodigits x_plus_mask = ((twodigits)x) + PyLong_MASK; @@ -61,7 +70,7 @@ maybe_small_long(PyLongObject *v) if (v && IS_MEDIUM_VALUE(v)) { stwodigits ival = medium_value(v); if (IS_SMALL_INT(ival)) { - _Py_DECREF_INT((PyObject *)v); + _Py_DECREF_INT(v); return (PyLongObject *)get_small_int((sdigit)ival); } } @@ -1824,7 +1833,7 @@ long_to_decimal_string_internal(PyObject *aa, #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS - _Py_DECREF_INT((PyObject *)scratch); + _Py_DECREF_INT(scratch); if (writer) { writer->pos += strlen; } @@ -3479,15 +3488,15 @@ k_mul(PyLongObject *a, PyLongObject *b) */ i = Py_SIZE(ret) - shift; /* # digits after shift */ (void)v_isub(ret->ob_digit + shift, i, t2->ob_digit, Py_SIZE(t2)); - _Py_DECREF_INT((PyObject *)t2); + _Py_DECREF_INT(t2); (void)v_isub(ret->ob_digit + shift, i, t1->ob_digit, Py_SIZE(t1)); - _Py_DECREF_INT((PyObject *)t1); + _Py_DECREF_INT(t1); /* 6. t3 <- (ah+al)(bh+bl), and add into result. */ if ((t1 = x_add(ah, al)) == NULL) goto fail; - _Py_DECREF_INT((PyObject *)ah); - _Py_DECREF_INT((PyObject *)al); + _Py_DECREF_INT(ah); + _Py_DECREF_INT(al); ah = al = NULL; if (a == b) { @@ -3498,13 +3507,13 @@ k_mul(PyLongObject *a, PyLongObject *b) Py_DECREF(t1); goto fail; } - _Py_DECREF_INT((PyObject *)bh); - _Py_DECREF_INT((PyObject *)bl); + _Py_DECREF_INT(bh); + _Py_DECREF_INT(bl); bh = bl = NULL; t3 = k_mul(t1, t2); - _Py_DECREF_INT((PyObject *)t1); - _Py_DECREF_INT((PyObject *)t2); + _Py_DECREF_INT(t1); + _Py_DECREF_INT(t2); if (t3 == NULL) goto fail; assert(Py_SIZE(t3) >= 0); @@ -3512,7 +3521,7 @@ k_mul(PyLongObject *a, PyLongObject *b) * See the (*) comment after this function. */ (void)v_iadd(ret->ob_digit + shift, i, t3->ob_digit, Py_SIZE(t3)); - _Py_DECREF_INT((PyObject *)t3); + _Py_DECREF_INT(t3); return long_normalize(ret); @@ -3617,13 +3626,13 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b) /* Add into result. */ (void)v_iadd(ret->ob_digit + nbdone, Py_SIZE(ret) - nbdone, product->ob_digit, Py_SIZE(product)); - _Py_DECREF_INT((PyObject *)product); + _Py_DECREF_INT(product); bsize -= nbtouse; nbdone += nbtouse; } - _Py_DECREF_INT((PyObject *)bslice); + _Py_DECREF_INT(bslice); return long_normalize(ret); fail: diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 6dcb23694d2fcf..f0be84bf0f86e1 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2520,7 +2520,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) } if (PyFloat_CheckExact(item)) { f_result += PyFloat_AS_DOUBLE(item); - _Py_DECREF_FLOAT(item); + _Py_DECREF_SPECIALIZED(item, (destructor)_PyFloat_ExactDealloc); continue; } if (PyLong_Check(item)) { diff --git a/Python/ceval.c b/Python/ceval.c index f73829c170a63b..0f2d4b6f1e657f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2024,8 +2024,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(BINARY_OP, hit); PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(prod); - _Py_DECREF_INT(right); - _Py_DECREF_INT(left); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); STACK_SHRINK(1); if (prod == NULL) { goto error; @@ -2043,8 +2043,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr ((PyFloatObject *)right)->ob_fval; PyObject *prod = PyFloat_FromDouble(dprod); SET_SECOND(prod); - _Py_DECREF_FLOAT(right); - _Py_DECREF_FLOAT(left); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyFloat_ExactDealloc); STACK_SHRINK(1); if (prod == NULL) { goto error; @@ -2060,8 +2060,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(BINARY_OP, hit); PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sub); - _Py_DECREF_INT(right); - _Py_DECREF_INT(left); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); STACK_SHRINK(1); if (sub == NULL) { goto error; @@ -2078,8 +2078,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; PyObject *sub = PyFloat_FromDouble(dsub); SET_SECOND(sub); - _Py_DECREF_FLOAT(right); - _Py_DECREF_FLOAT(left); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyFloat_ExactDealloc); STACK_SHRINK(1); if (sub == NULL) { goto error; @@ -2096,8 +2096,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *res = PyUnicode_Concat(left, right); STACK_SHRINK(1); SET_TOP(res); - _Py_DECREF_STR(left); - _Py_DECREF_STR(right); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyUnicode_ExactDealloc); if (TOP() == NULL) { goto error; } @@ -2122,10 +2122,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DEOPT_IF(var != left, BINARY_OP); STAT_INC(BINARY_OP, hit); GETLOCAL(next_oparg) = NULL; - _Py_DECREF_STR(left); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyUnicode_ExactDealloc); STACK_SHRINK(1); PyUnicode_Append(&TOP(), right); - _Py_DECREF_STR(right); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyUnicode_ExactDealloc); if (TOP() == NULL) { goto error; } @@ -2142,8 +2142,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr ((PyFloatObject *)right)->ob_fval; PyObject *sum = PyFloat_FromDouble(dsum); SET_SECOND(sum); - _Py_DECREF_FLOAT(right); - _Py_DECREF_FLOAT(left); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyFloat_ExactDealloc); STACK_SHRINK(1); if (sum == NULL) { goto error; @@ -2159,8 +2159,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(BINARY_OP, hit); PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sum); - _Py_DECREF_INT(right); - _Py_DECREF_INT(left); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); STACK_SHRINK(1); if (sum == NULL) { goto error; @@ -2218,7 +2218,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(res != NULL); Py_INCREF(res); STACK_SHRINK(1); - _Py_DECREF_INT(sub); + _Py_DECREF_SPECIALIZED(sub, (destructor)_PyLong_ExactDealloc); SET_TOP(res); Py_DECREF(list); DISPATCH(); @@ -2241,7 +2241,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(res != NULL); Py_INCREF(res); STACK_SHRINK(1); - _Py_DECREF_INT(sub); + _Py_DECREF_SPECIALIZED(sub, (destructor)_PyLong_ExactDealloc); SET_TOP(res); Py_DECREF(tuple); DISPATCH(); @@ -2372,7 +2372,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STACK_SHRINK(3); assert(old_value != NULL); Py_DECREF(old_value); - _Py_DECREF_INT(sub); + _Py_DECREF_SPECIALIZED(sub, (destructor)_PyLong_ExactDealloc); Py_DECREF(list); DISPATCH(); } @@ -3211,12 +3211,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto error; } str = _PyUnicode_JoinArray(empty, stack_pointer - oparg, oparg); - _Py_DECREF_STR(empty); + _Py_DECREF_SPECIALIZED(empty, (destructor)_PyUnicode_ExactDealloc); if (str == NULL) goto error; while (--oparg >= 0) { PyObject *item = POP(); - _Py_DECREF_STR(item); + _Py_DECREF_SPECIALIZED(item, (destructor)_PyUnicode_ExactDealloc); } PUSH(str); DISPATCH(); @@ -3741,8 +3741,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(COMPARE_OP, hit); NEXTOPARG(); STACK_SHRINK(2); - _Py_DECREF_FLOAT(left); - _Py_DECREF_FLOAT(right); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyFloat_ExactDealloc); assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE); int jump = (1 << (sign + 1)) & when_to_jump_mask; if (!jump) { @@ -3774,8 +3774,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr int sign = (ileft > iright) - (ileft < iright); NEXTOPARG(); STACK_SHRINK(2); - _Py_DECREF_INT(left); - _Py_DECREF_INT(right); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE); int jump = (1 << (sign + 1)) & when_to_jump_mask; if (!jump) { @@ -3808,8 +3808,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr NEXTOPARG(); assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE); STACK_SHRINK(2); - _Py_DECREF_STR(left); - _Py_DECREF_STR(right); + _Py_DECREF_SPECIALIZED(left, (destructor)_PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, (destructor)_PyUnicode_ExactDealloc); assert(res == 0 || res == 1); assert(invert == 0 || invert == 1); int jump = res ^ invert; From 8edfa4d8031b4f5d5c758fa07a13e82765271651 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Wed, 26 Jan 2022 15:38:59 -0500 Subject: [PATCH 06/14] Hide more functions --- Include/internal/pycore_floatobject.h | 3 +-- Include/internal/pycore_long.h | 2 +- Include/internal/pycore_object.h | 31 ++++++++++++++++++++++++++ Include/object.h | 32 --------------------------- Modules/mathmodule.c | 1 + 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index e553d9ae72aed8..1da711e26bd73d 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -38,8 +38,7 @@ struct _Py_float_state { #endif }; -PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyFloatObject *op); - +void _PyFloat_ExactDealloc(PyFloatObject *op); /* _PyFloat_{Pack,Unpack}{4,8} * diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index a0ed2f471d8b8f..a41217f165cf1b 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -67,7 +67,7 @@ PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( int base, int alternate); -PyAPI_FUNC(void) _PyLong_ExactDealloc(PyLongObject *op); +void _PyLong_ExactDealloc(PyLongObject *op); #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index be308cd25d710c..6c8790e9d06395 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -24,6 +24,37 @@ extern "C" { .ob_size = size, \ } +static inline void +_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) +{ +#ifdef Py_REF_DEBUG + _Py_RefTotal--; +#endif + if (--op->ob_refcnt != 0) { + assert(op->ob_refcnt > 0); + } + else { +#ifdef Py_TRACE_REFS + _Py_ForgetReference(op); +#endif + destruct(op); + } +} + +static inline void +_Py_DECREF_IMMORTAL(PyObject *op) +{ +#ifdef Py_REF_DEBUG + _Py_RefTotal--; +#endif + op->ob_refcnt--; +#ifdef Py_DEBUG + if (op->ob_refcnt <= 0) { + // Calls _Py_FatalRefcountError for None, True, and False + _Py_Dealloc(op); + } +#endif +} PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type); PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); diff --git a/Include/object.h b/Include/object.h index 50a0a1a8211639..4fd16616ce705e 100644 --- a/Include/object.h +++ b/Include/object.h @@ -524,37 +524,6 @@ static inline void _Py_DECREF( # define Py_DECREF(op) _Py_DECREF(_PyObject_CAST(op)) #endif -static inline void -_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) -{ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif - if (--op->ob_refcnt != 0) { - assert(op->ob_refcnt > 0); - } - else { -#ifdef Py_TRACE_REFS - _Py_ForgetReference(op); -#endif - destruct(op); - } -} - -static inline void -_Py_DECREF_IMMORTAL(PyObject *op) -{ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif - op->ob_refcnt--; -#ifdef Py_DEBUG - if (op->ob_refcnt <= 0) { - // Calls _Py_FatalRefcountError for None, True, and False - _Py_Dealloc(op); - } -#endif -} /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear * and tp_dealloc implementations. @@ -653,7 +622,6 @@ Don't forget to apply Py_INCREF() when returning this value!!! PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ #define Py_None (&_Py_NoneStruct) - // Test if an object is the None singleton, the same as "x is None" in Python. PyAPI_FUNC(int) Py_IsNone(PyObject *x); #define Py_IsNone(x) Py_Is((x), Py_None) diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 058dbf698cabf0..424f61d541d3e7 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -62,6 +62,7 @@ raised for division by zero and mod by zero. #include "pycore_dtoa.h" // _Py_dg_infinity() #include "pycore_long.h" // _PyLong_GetZero(), _PyLong_ExactDealloc #include "pycore_floatobject.h" // _PyFloat_ExactDealloc +#include "pycore_object.h" // _Py_DECREF_SPECIALIZED /* For DBL_EPSILON in _math.h */ #include /* For _Py_log1p with workarounds for buggy handling of zeros. */ From d095c43d1d69d0584888e3ec9b50429f8a794ea0 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Wed, 26 Jan 2022 19:12:03 -0500 Subject: [PATCH 07/14] Use PyAPI_FUNC again --- Include/internal/pycore_floatobject.h | 2 +- Include/internal/pycore_long.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 1da711e26bd73d..f4d50dc6e8c245 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -38,7 +38,7 @@ struct _Py_float_state { #endif }; -void _PyFloat_ExactDealloc(PyFloatObject *op); +PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyFloatObject *op); /* _PyFloat_{Pack,Unpack}{4,8} * diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index a41217f165cf1b..a0ed2f471d8b8f 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -67,7 +67,7 @@ PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( int base, int alternate); -void _PyLong_ExactDealloc(PyLongObject *op); +PyAPI_FUNC(void) _PyLong_ExactDealloc(PyLongObject *op); #ifdef __cplusplus } From fa6a26aade7c144df876f6b675758ff311508fe1 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Thu, 27 Jan 2022 15:20:27 -0500 Subject: [PATCH 08/14] Use PyObject* to avoid casts --- Include/internal/pycore_floatobject.h | 2 +- Include/internal/pycore_long.h | 2 +- Include/internal/pycore_unicodeobject.h | 2 +- Objects/floatobject.c | 10 +++-- Objects/longobject.c | 2 +- Objects/unicodeobject.c | 4 +- Python/ceval.c | 54 ++++++++++++------------- 7 files changed, 39 insertions(+), 37 deletions(-) diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index f4d50dc6e8c245..120dda6553090f 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -38,7 +38,7 @@ struct _Py_float_state { #endif }; -PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyFloatObject *op); +PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyObject *op); /* _PyFloat_{Pack,Unpack}{4,8} * diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index a0ed2f471d8b8f..c24ed0a38f1c85 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -67,7 +67,7 @@ PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( int base, int alternate); -PyAPI_FUNC(void) _PyLong_ExactDealloc(PyLongObject *op); +PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *op); #ifdef __cplusplus } diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 3eb5e359231a98..cc355000ba2951 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -10,7 +10,7 @@ extern "C" { #include "pycore_fileutils.h" // _Py_error_handler -void _PyUnicode_ExactDealloc(PyUnicodeObject *op); +void _PyUnicode_ExactDealloc(PyObject *op); /* runtime lifecycle */ diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 6e5bb3d455e058..249e169bf95c28 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -239,9 +239,10 @@ PyFloat_FromString(PyObject *v) } void -_PyFloat_ExactDealloc(PyFloatObject *op) +_PyFloat_ExactDealloc(PyObject *obj) { - assert(PyFloat_CheckExact(op)); + assert(PyFloat_CheckExact(obj)); + PyFloatObject *op = (PyFloatObject *)obj; #if PyFloat_MAXFREELIST > 0 struct _Py_float_state *state = get_float_state(); #ifdef Py_DEBUG @@ -261,8 +262,9 @@ _PyFloat_ExactDealloc(PyFloatObject *op) } static void -float_dealloc(PyFloatObject *op) +float_dealloc(PyObject *op) { + assert(PyFloat_Check(op)); #if PyFloat_MAXFREELIST > 0 if (PyFloat_CheckExact(op)) { _PyFloat_ExactDealloc(op); @@ -270,7 +272,7 @@ float_dealloc(PyFloatObject *op) else #endif { - Py_TYPE(op)->tp_free((PyObject *)op); + Py_TYPE(op)->tp_free(op); } } diff --git a/Objects/longobject.c b/Objects/longobject.c index 3be90da34ff239..9352d1d2f89069 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -5753,7 +5753,7 @@ int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, } void -_PyLong_ExactDealloc(PyLongObject *op) +_PyLong_ExactDealloc(PyObject *op) { assert(PyLong_CheckExact(op)); PyObject_Del(op); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 659db01c38a84f..a253dd521eb688 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15464,10 +15464,10 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) } void -_PyUnicode_ExactDealloc(PyUnicodeObject *op) +_PyUnicode_ExactDealloc(PyObject *op) { assert(PyUnicode_CheckExact(op)); - unicode_dealloc((PyObject *)op); + unicode_dealloc(op); } PyDoc_STRVAR(unicode_doc, diff --git a/Python/ceval.c b/Python/ceval.c index 0f2d4b6f1e657f..b6b580f45a99d6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2024,8 +2024,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(BINARY_OP, hit); PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(prod); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyLong_ExactDealloc); STACK_SHRINK(1); if (prod == NULL) { goto error; @@ -2043,8 +2043,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr ((PyFloatObject *)right)->ob_fval; PyObject *prod = PyFloat_FromDouble(dprod); SET_SECOND(prod); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, (destructor)_PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); STACK_SHRINK(1); if (prod == NULL) { goto error; @@ -2060,8 +2060,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(BINARY_OP, hit); PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sub); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyLong_ExactDealloc); STACK_SHRINK(1); if (sub == NULL) { goto error; @@ -2078,8 +2078,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; PyObject *sub = PyFloat_FromDouble(dsub); SET_SECOND(sub); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, (destructor)_PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); STACK_SHRINK(1); if (sub == NULL) { goto error; @@ -2096,8 +2096,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr PyObject *res = PyUnicode_Concat(left, right); STACK_SHRINK(1); SET_TOP(res); - _Py_DECREF_SPECIALIZED(left, (destructor)_PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (TOP() == NULL) { goto error; } @@ -2122,10 +2122,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr DEOPT_IF(var != left, BINARY_OP); STAT_INC(BINARY_OP, hit); GETLOCAL(next_oparg) = NULL; - _Py_DECREF_SPECIALIZED(left, (destructor)_PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); STACK_SHRINK(1); PyUnicode_Append(&TOP(), right); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (TOP() == NULL) { goto error; } @@ -2142,8 +2142,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr ((PyFloatObject *)right)->ob_fval; PyObject *sum = PyFloat_FromDouble(dsum); SET_SECOND(sum); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, (destructor)_PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); STACK_SHRINK(1); if (sum == NULL) { goto error; @@ -2159,8 +2159,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(BINARY_OP, hit); PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sum); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyLong_ExactDealloc); STACK_SHRINK(1); if (sum == NULL) { goto error; @@ -2218,7 +2218,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(res != NULL); Py_INCREF(res); STACK_SHRINK(1); - _Py_DECREF_SPECIALIZED(sub, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(sub, _PyLong_ExactDealloc); SET_TOP(res); Py_DECREF(list); DISPATCH(); @@ -2241,7 +2241,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr assert(res != NULL); Py_INCREF(res); STACK_SHRINK(1); - _Py_DECREF_SPECIALIZED(sub, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(sub, _PyLong_ExactDealloc); SET_TOP(res); Py_DECREF(tuple); DISPATCH(); @@ -2372,7 +2372,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STACK_SHRINK(3); assert(old_value != NULL); Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(sub, _PyLong_ExactDealloc); Py_DECREF(list); DISPATCH(); } @@ -3211,12 +3211,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr goto error; } str = _PyUnicode_JoinArray(empty, stack_pointer - oparg, oparg); - _Py_DECREF_SPECIALIZED(empty, (destructor)_PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(empty, _PyUnicode_ExactDealloc); if (str == NULL) goto error; while (--oparg >= 0) { PyObject *item = POP(); - _Py_DECREF_SPECIALIZED(item, (destructor)_PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(item, _PyUnicode_ExactDealloc); } PUSH(str); DISPATCH(); @@ -3741,8 +3741,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr STAT_INC(COMPARE_OP, hit); NEXTOPARG(); STACK_SHRINK(2); - _Py_DECREF_SPECIALIZED(left, (destructor)_PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE); int jump = (1 << (sign + 1)) & when_to_jump_mask; if (!jump) { @@ -3774,8 +3774,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr int sign = (ileft > iright) - (ileft < iright); NEXTOPARG(); STACK_SHRINK(2); - _Py_DECREF_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyLong_ExactDealloc); assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE); int jump = (1 << (sign + 1)) & when_to_jump_mask; if (!jump) { @@ -3808,8 +3808,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr NEXTOPARG(); assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE); STACK_SHRINK(2); - _Py_DECREF_SPECIALIZED(left, (destructor)_PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, (destructor)_PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); assert(res == 0 || res == 1); assert(invert == 0 || invert == 1); int jump = res ^ invert; From 35c0d25eba64c2ba0a095ff82e788762ee6a1bbc Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Thu, 27 Jan 2022 16:44:31 -0500 Subject: [PATCH 09/14] Remove PyAPI_FUNC and mathmodule optimizations --- Include/internal/pycore_floatobject.h | 2 +- Include/internal/pycore_long.h | 2 +- Modules/mathmodule.c | 14 +++++--------- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 120dda6553090f..6d7b9ade791a5b 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -38,7 +38,7 @@ struct _Py_float_state { #endif }; -PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyObject *op); +void _PyFloat_ExactDealloc(PyObject *op); /* _PyFloat_{Pack,Unpack}{4,8} * diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index c24ed0a38f1c85..ced29cbc12c217 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -67,7 +67,7 @@ PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( int base, int alternate); -PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *op); +void _PyLong_ExactDealloc(PyObject *op); #ifdef __cplusplus } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 424f61d541d3e7..0c7d4de0686213 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -60,9 +60,7 @@ raised for division by zero and mod by zero. #include "pycore_bitutils.h" // _Py_bit_length() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_dtoa.h" // _Py_dg_infinity() -#include "pycore_long.h" // _PyLong_GetZero(), _PyLong_ExactDealloc -#include "pycore_floatobject.h" // _PyFloat_ExactDealloc -#include "pycore_object.h" // _Py_DECREF_SPECIALIZED +#include "pycore_long.h" // _PyLong_GetZero() /* For DBL_EPSILON in _math.h */ #include /* For _Py_log1p with workarounds for buggy handling of zeros. */ @@ -3141,7 +3139,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) long i_result = PyLong_AsLongAndOverflow(result, &overflow); /* If this already overflowed, don't even enter the loop. */ if (overflow == 0) { - _Py_DECREF_SPECIALIZED(result, (destructor)_PyLong_ExactDealloc); + Py_DECREF(result); result = NULL; } /* Loop over all the items in the iterable until we finish, we overflow @@ -3160,8 +3158,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) if (overflow == 0 && !_check_long_mult_overflow(i_result, b)) { long x = i_result * b; i_result = x; - _Py_DECREF_SPECIALIZED(item, - (destructor)_PyLong_ExactDealloc); + Py_DECREF(item); continue; } } @@ -3190,7 +3187,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) */ if (PyFloat_CheckExact(result)) { double f_result = PyFloat_AS_DOUBLE(result); - _Py_DECREF_SPECIALIZED(result, (destructor)_PyFloat_ExactDealloc); + Py_DECREF(result); result = NULL; while(result == NULL) { item = PyIter_Next(iter); @@ -3203,8 +3200,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) } if (PyFloat_CheckExact(item)) { f_result *= PyFloat_AS_DOUBLE(item); - _Py_DECREF_SPECIALIZED(item, - (destructor)_PyFloat_ExactDealloc); + Py_DECREF(item); continue; } if (PyLong_CheckExact(item)) { From feebdb32d1f414dd63d5a91cf727ce2bd6058f10 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Fri, 28 Jan 2022 13:23:15 -0500 Subject: [PATCH 10/14] Use _Py_FataRefcountError() directly. Move it to the same header file. --- Include/internal/pycore_object.h | 10 +++++++--- Include/internal/pycore_pyerrors.h | 7 ------- Objects/boolobject.c | 2 +- Objects/object.c | 2 +- Objects/tupleobject.c | 3 +-- Objects/unicodeobject.c | 3 +-- 6 files changed, 11 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 6c8790e9d06395..ccf9e76feee3e5 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -12,7 +12,6 @@ extern "C" { #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_pystate.h" // _PyInterpreterState_GET() - #define _PyObject_IMMORTAL_INIT(type) \ { \ .ob_refcnt = 999999999, \ @@ -24,6 +23,12 @@ extern "C" { .ob_size = size, \ } +PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( + const char *func, + const char *message); + +#define _Py_FatalRefcountError(message) _Py_FatalRefcountErrorFunc(__func__, message) + static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { @@ -50,8 +55,7 @@ _Py_DECREF_IMMORTAL(PyObject *op) op->ob_refcnt--; #ifdef Py_DEBUG if (op->ob_refcnt <= 0) { - // Calls _Py_FatalRefcountError for None, True, and False - _Py_Dealloc(op); + _Py_FatalRefcountError("deallocating a singleton"); } #endif } diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index e3c445ba5d9267..66f37942ef916a 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -100,13 +100,6 @@ extern PyObject* _Py_Offer_Suggestions(PyObject* exception); PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, Py_ssize_t max_cost); -PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( - const char *func, - const char *message); - -#define _Py_FatalRefcountError(message) _Py_FatalRefcountErrorFunc(__func__, message) - - #ifdef __cplusplus } #endif diff --git a/Objects/boolobject.c b/Objects/boolobject.c index 53f81926057974..41ebfcef80a397 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -1,7 +1,7 @@ /* Boolean type, a subtype of int */ #include "Python.h" -#include "pycore_pyerrors.h" // _Py_FatalRefcountError() +#include "pycore_object.h" // _Py_FatalRefcountError() /* We define bool_repr to return "False" or "True" */ diff --git a/Objects/object.c b/Objects/object.c index 3082e70e7e230b..0ce21f24cf8229 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -9,7 +9,7 @@ #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_namespace.h" // _PyNamespace_Type -#include "pycore_object.h" // _PyType_CheckConsistency() +#include "pycore_object.h" // _PyType_CheckConsistency(), _Py_FatalRefcountError() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 86f541a96a5a14..c1839fbb7840b3 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -5,8 +5,7 @@ #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_pyerrors.h" // _Py_FatalRefcountError() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError() #include "pycore_tuple.h" // struct _Py_tuple_state() /*[clinic input] diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a253dd521eb688..d66f3bb487a0c6 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -47,9 +47,8 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.fs_codec #include "pycore_long.h" // _PyLong_FormatWriter() -#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError() #include "pycore_pathconfig.h" // _Py_DumpPathConfig() -#include "pycore_pyerrors.h" // _Py_FatalRefcountError() #include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI From 05b050bac598a663353699107cf4a73f8ffa32ab Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Fri, 28 Jan 2022 23:48:10 -0500 Subject: [PATCH 11/14] remove (destructor) cast in bltinmodule and longobject --- Objects/longobject.c | 3 +-- Python/bltinmodule.c | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 9352d1d2f89069..484e061e3dbb4e 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -43,8 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, - (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED((PyObject *)op, _PyLong_ExactDealloc); } static inline int diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f0be84bf0f86e1..91de04a9f6e00e 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2520,7 +2520,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) } if (PyFloat_CheckExact(item)) { f_result += PyFloat_AS_DOUBLE(item); - _Py_DECREF_SPECIALIZED(item, (destructor)_PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(item, _PyFloat_ExactDealloc); continue; } if (PyLong_Check(item)) { From 0d6a59823869930c2b2f6b72c12f648687c20d76 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Fri, 11 Feb 2022 11:47:24 -0500 Subject: [PATCH 12/14] Add newline at end of NEWS entry --- .../Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst index 08c2ccb976fe6e..e19ce0a243c6c8 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst @@ -1 +1 @@ -Add type-specialized versions of the ``Py_DECREF()``, and use them for ``float``, ``int``, ``str``, ``bool``, and ``None`` to avoid pointer-chasing at runtime where types are known at C compile time. \ No newline at end of file +Add type-specialized versions of the ``Py_DECREF()``, and use them for ``float``, ``int``, ``str``, ``bool``, and ``None`` to avoid pointer-chasing at runtime where types are known at C compile time. From ad146ef86e5cae1cac0affaefd3eabda37bd6a87 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Tue, 12 Apr 2022 01:13:23 -0400 Subject: [PATCH 13/14] naming: IMMORTAL --> NO_DEALLOC --- Include/internal/pycore_object.h | 4 ++-- Python/ceval.c | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index c3ba4bea2f3435..f022f8246989d1 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -49,7 +49,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) } static inline void -_Py_DECREF_IMMORTAL(PyObject *op) +_Py_DECREF_NO_DEALLOC(PyObject *op) { #ifdef Py_REF_DEBUG _Py_RefTotal--; @@ -57,7 +57,7 @@ _Py_DECREF_IMMORTAL(PyObject *op) op->ob_refcnt--; #ifdef Py_DEBUG if (op->ob_refcnt <= 0) { - _Py_FatalRefcountError("deallocating a singleton"); + _Py_FatalRefcountError("Expected a positive remaining refcount"); } #endif } diff --git a/Python/ceval.c b/Python/ceval.c index 4a2f386430d00f..b9a07bcd696329 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2018,7 +2018,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int * that the string is safe to mutate. */ assert(Py_REFCNT(left) >= 2); - Py_DECREF(left); // XXX never need to dealloc + _Py_DECREF_NO_DEALLOC(left); STACK_SHRINK(2); PyUnicode_Append(target_local, right); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); @@ -3940,11 +3940,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PREDICTED(POP_JUMP_BACKWARD_IF_FALSE); PyObject *cond = POP(); if (Py_IsTrue(cond)) { - _Py_DECREF_IMMORTAL(cond); + _Py_DECREF_NO_DEALLOC(cond); DISPATCH(); } if (Py_IsFalse(cond)) { - _Py_DECREF_IMMORTAL(cond); + _Py_DECREF_NO_DEALLOC(cond); JUMPBY(-oparg); CHECK_EVAL_BREAKER(); DISPATCH(); @@ -3966,10 +3966,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PREDICTED(POP_JUMP_FORWARD_IF_FALSE); PyObject *cond = POP(); if (Py_IsTrue(cond)) { - _Py_DECREF_IMMORTAL(cond); + _Py_DECREF_NO_DEALLOC(cond); } else if (Py_IsFalse(cond)) { - _Py_DECREF_IMMORTAL(cond); + _Py_DECREF_NO_DEALLOC(cond); JUMPBY(oparg); } else { @@ -3989,11 +3989,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(POP_JUMP_BACKWARD_IF_TRUE) { PyObject *cond = POP(); if (Py_IsFalse(cond)) { - _Py_DECREF_IMMORTAL(cond); + _Py_DECREF_NO_DEALLOC(cond); DISPATCH(); } if (Py_IsTrue(cond)) { - _Py_DECREF_IMMORTAL(cond); + _Py_DECREF_NO_DEALLOC(cond); JUMPBY(-oparg); CHECK_EVAL_BREAKER(); DISPATCH(); @@ -4014,10 +4014,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(POP_JUMP_FORWARD_IF_TRUE) { PyObject *cond = POP(); if (Py_IsFalse(cond)) { - _Py_DECREF_IMMORTAL(cond); + _Py_DECREF_NO_DEALLOC(cond); } else if (Py_IsTrue(cond)) { - _Py_DECREF_IMMORTAL(cond); + _Py_DECREF_NO_DEALLOC(cond); JUMPBY(oparg); } else { @@ -4042,7 +4042,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int CHECK_EVAL_BREAKER(); DISPATCH(); } - _Py_DECREF_IMMORTAL(value); + _Py_DECREF_NO_DEALLOC(value); DISPATCH(); } @@ -4058,7 +4058,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(POP_JUMP_BACKWARD_IF_NONE) { PyObject *value = POP(); if (Py_IsNone(value)) { - _Py_DECREF_IMMORTAL(value); + _Py_DECREF_NO_DEALLOC(value); JUMPBY(-oparg); CHECK_EVAL_BREAKER(); } @@ -4071,7 +4071,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(POP_JUMP_FORWARD_IF_NONE) { PyObject *value = POP(); if (Py_IsNone(value)) { - _Py_DECREF_IMMORTAL(value); + _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { @@ -4085,7 +4085,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int int err; if (Py_IsTrue(cond)) { STACK_SHRINK(1); - _Py_DECREF_IMMORTAL(cond); + _Py_DECREF_NO_DEALLOC(cond); DISPATCH(); } if (Py_IsFalse(cond)) { @@ -4109,7 +4109,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int int err; if (Py_IsFalse(cond)) { STACK_SHRINK(1); - _Py_DECREF_IMMORTAL(cond); + _Py_DECREF_NO_DEALLOC(cond); DISPATCH(); } if (Py_IsTrue(cond)) { From 3dd308a8d03aacd641a95b0c7126f66134e7d908 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Tue, 19 Apr 2022 11:18:30 -0400 Subject: [PATCH 14/14] Remove _PyLong_ExactDealloc --- Include/internal/pycore_long.h | 2 -- Objects/longobject.c | 11 ++--------- Python/ceval.c | 22 +++++++++++----------- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index ae0ab703dde792..a33762402491b7 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -73,8 +73,6 @@ PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( int base, int alternate); -void _PyLong_ExactDealloc(PyObject *op); - #ifdef __cplusplus } #endif diff --git a/Objects/longobject.c b/Objects/longobject.c index ba5ab8f2b89824..c104dcc14f986e 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -40,7 +40,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED((PyObject *)op, PyObject_Free); } static inline int @@ -5860,13 +5860,6 @@ int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, return long_obj; } -void -_PyLong_ExactDealloc(PyObject *op) -{ - assert(PyLong_CheckExact(op)); - PyObject_Del(op); -} - static PyObject * long_long_meth(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -6008,7 +6001,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_init */ 0, /* tp_alloc */ long_new, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; static PyTypeObject Int_InfoType; diff --git a/Python/ceval.c b/Python/ceval.c index 4f705f853405c5..45754ffbe7cb4a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1977,8 +1977,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int STAT_INC(BINARY_OP, hit); PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(prod); - _Py_DECREF_SPECIALIZED(right, _PyLong_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, PyObject_Free); + _Py_DECREF_SPECIALIZED(left, PyObject_Free); STACK_SHRINK(1); if (prod == NULL) { goto error; @@ -2017,8 +2017,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int STAT_INC(BINARY_OP, hit); PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sub); - _Py_DECREF_SPECIALIZED(right, _PyLong_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, PyObject_Free); + _Py_DECREF_SPECIALIZED(left, PyObject_Free); STACK_SHRINK(1); if (sub == NULL) { goto error; @@ -2132,8 +2132,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int STAT_INC(BINARY_OP, hit); PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sum); - _Py_DECREF_SPECIALIZED(right, _PyLong_ExactDealloc); - _Py_DECREF_SPECIALIZED(left, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, PyObject_Free); + _Py_DECREF_SPECIALIZED(left, PyObject_Free); STACK_SHRINK(1); if (sum == NULL) { goto error; @@ -2192,7 +2192,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(res != NULL); Py_INCREF(res); STACK_SHRINK(1); - _Py_DECREF_SPECIALIZED(sub, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(sub, PyObject_Free); SET_TOP(res); Py_DECREF(list); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); @@ -2217,7 +2217,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(res != NULL); Py_INCREF(res); STACK_SHRINK(1); - _Py_DECREF_SPECIALIZED(sub, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(sub, PyObject_Free); SET_TOP(res); Py_DECREF(tuple); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); @@ -2359,7 +2359,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int STACK_SHRINK(3); assert(old_value != NULL); Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(sub, PyObject_Free); Py_DECREF(list); JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); NOTRACE_DISPATCH(); @@ -3795,8 +3795,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); NEXTOPARG(); STACK_SHRINK(2); - _Py_DECREF_SPECIALIZED(left, _PyLong_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, PyObject_Free); + _Py_DECREF_SPECIALIZED(right, PyObject_Free); assert(opcode == POP_JUMP_FORWARD_IF_FALSE || opcode == POP_JUMP_BACKWARD_IF_FALSE || opcode == POP_JUMP_FORWARD_IF_TRUE ||