From fb1bc5e4642ef2039da5596b2051f23c756fc061 Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Thu, 13 Jun 2019 17:18:32 -0400 Subject: [PATCH] Manage memory lifetime for all type-related objects. Remove internal usages of PyMethodDef and PyGetSetDef. For PyMethodDef, change PyCFunctionObject to replace the PyMethodDef* member with a PyCFunctionBase member. The PyCFunctionBase is a new struct to hold the managed values of a PyMethodDef. This type is shared between PyCFunction and the various callable descriptor objects. A PyCFunctionBase is like a PyMethodDef but replaces the char* members with PyObject* members. For PyGetSetDef, inline the members on the resulting PyGetSetDescrObject, replacing all char* members with PyObject* members. The memory for the closure is *not* managed, adding support for that would likely require an API change and can be done in a future change. For the tp_name field, instead of setting it directly to the value of PyType_Spec.name, set it to the result of PyUnicode_AsUTF8(ht_name), where ht_name is the PyUnicode object created from the original spec name. This is the same trick used to properly manage this pointer for heap types when the __name__ is reassigned. --- Doc/c-api/type.rst | 3 + Include/bltinmodule.h | 4 + Include/cpython/object.h | 16 ++- Include/descrobject.h | 12 +- Include/methodobject.h | 34 +++-- Include/structmember.h | 5 + Lib/test/test_cprofile.py | 64 ++++----- Lib/test/test_sys.py | 8 +- .../2019-06-18-00-55-12.bpo-37270.2z98X7.rst | 6 + Modules/_lsprof.c | 38 +++-- Objects/abstract.c | 3 +- Objects/call.c | 73 +++++----- Objects/descrobject.c | 131 ++++++++++++----- Objects/methodobject.c | 133 +++++++++++------ Objects/structseq.c | 134 +++++++++++++----- Objects/typeobject.c | 76 +++++----- Python/bltinmodule.c | 6 +- Python/ceval.c | 2 +- Python/structmember.c | 70 +++++++-- Tools/gdb/libpython.py | 24 ++-- 20 files changed, 547 insertions(+), 295 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2019-06-18-00-55-12.bpo-37270.2z98X7.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 8f8367ab77c8c4..74271d8664b195 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -128,6 +128,9 @@ The following functions and structs are used to create This function calls :c:func:`PyType_Ready` on the new type. + Until Python 3.9, the memory backing the ``Py_tp_methods``, ``Py_tp_getset``, + and :c:member:`PyType_Spec.name` must outlive the returned type. + .. versionadded:: 3.3 .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) diff --git a/Include/bltinmodule.h b/Include/bltinmodule.h index 868c9e6443bfc1..232d3f02643aff 100644 --- a/Include/bltinmodule.h +++ b/Include/bltinmodule.h @@ -8,6 +8,10 @@ PyAPI_DATA(PyTypeObject) PyFilter_Type; PyAPI_DATA(PyTypeObject) PyMap_Type; PyAPI_DATA(PyTypeObject) PyZip_Type; +#ifdef Py_BUILD_CORE +PyObject* _PyBuiltin_Print(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/cpython/object.h b/Include/cpython/object.h index fd4e77103f01b8..4401aa00cad22a 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -282,12 +282,20 @@ typedef struct _heaptypeobject { PyBufferProcs as_buffer; PyObject *ht_name, *ht_slots, *ht_qualname; struct _dictkeysobject *ht_cached_keys; - /* here are optional user slots, followed by the members. */ + /* here are optional user slots, followed by the offsets of the object members. */ } PyHeapTypeObject; -/* access macro to the members which are floating "behind" the object */ -#define PyHeapType_GET_MEMBERS(etype) \ - ((PyMemberDef *)(((char *)etype) + Py_TYPE(etype)->tp_basicsize)) +#ifdef Py_BUILD_CORE +typedef struct { + Py_ssize_t offset; + int flags; +} _PyObject_MemberSlot; + +/* access macro to the offsets of the object type members which are floating + "behind" the object */ +#define _PyHeapType_GET_OBJECT_MEMBER_OFFSETS(etype) \ + ((_PyObject_MemberSlot *)(((char *)etype) + Py_TYPE(etype)->tp_basicsize)) +#endif PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); diff --git a/Include/descrobject.h b/Include/descrobject.h index d7114852c1e210..7da925ff91adc8 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -52,18 +52,24 @@ typedef struct { typedef struct { PyDescr_COMMON; - PyMethodDef *d_method; + PyCFunctionBase d_base; vectorcallfunc vectorcall; } PyMethodDescrObject; typedef struct { PyDescr_COMMON; - struct PyMemberDef *d_member; + int d_member_type; + Py_ssize_t d_offset; + int d_flags; + PyObject *d_doc; } PyMemberDescrObject; typedef struct { PyDescr_COMMON; - PyGetSetDef *d_getset; + getter d_get; + setter d_set; + PyObject *d_doc; + void *d_closure; } PyGetSetDescrObject; typedef struct { diff --git a/Include/methodobject.h b/Include/methodobject.h index e92adde7bf6bd7..1b880ead3d9716 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -32,12 +32,12 @@ PyAPI_FUNC(int) PyCFunction_GetFlags(PyObject *); done, so use with care. */ #ifndef Py_LIMITED_API #define PyCFunction_GET_FUNCTION(func) \ - (((PyCFunctionObject *)func) -> m_ml -> ml_meth) + (((PyCFunctionObject *)func) -> m_base.meth) #define PyCFunction_GET_SELF(func) \ - (((PyCFunctionObject *)func) -> m_ml -> ml_flags & METH_STATIC ? \ + (((PyCFunctionObject *)func) -> m_base.flags & METH_STATIC ? \ NULL : ((PyCFunctionObject *)func) -> m_self) #define PyCFunction_GET_FLAGS(func) \ - (((PyCFunctionObject *)func) -> m_ml -> ml_flags) + (((PyCFunctionObject *)func) -> m_base.flags) #endif PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *); @@ -99,24 +99,36 @@ PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *, #endif #ifndef Py_LIMITED_API +typedef struct { + PyObject *name; + PyCFunction meth; + PyObject *doc; + PyObject *signature; + int flags; +} PyCFunctionBase; + typedef struct { PyObject_HEAD - PyMethodDef *m_ml; /* Description of the C function to call */ - PyObject *m_self; /* Passed as 'self' arg to the C func, can be NULL */ - PyObject *m_module; /* The __module__ attribute, can be anything */ - PyObject *m_weakreflist; /* List of weak references */ + PyCFunctionBase m_base; + PyObject *m_self; /* Passed as 'self' arg to the C func, can be NULL */ + PyObject *m_module; /* The __module__ attribute, can be anything */ + PyObject *m_weakreflist; /* List of weak references */ vectorcallfunc vectorcall; } PyCFunctionObject; -PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallDict( - PyMethodDef *method, +PyAPI_FUNC(int) _PyCFunctionBase_FromMethodDef(PyCFunctionBase *, PyMethodDef *); +PyAPI_FUNC(void) _PyCFunctionBase_Clear(PyCFunctionBase *); + +PyAPI_FUNC(PyObject *) _PyCFunction_NewFromBase(PyCFunctionBase *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyCFunctionBase_RawFastCallDict( + PyCFunctionBase *func, PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); -PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallKeywords( - PyMethodDef *method, +PyAPI_FUNC(PyObject *) _PyCFunctionBase_RawFastCallKeywords( + PyCFunctionBase *func, PyObject *self, PyObject *const *args, Py_ssize_t nargs, diff --git a/Include/structmember.h b/Include/structmember.h index b54f7081f458dd..7132cb70e273f9 100644 --- a/Include/structmember.h +++ b/Include/structmember.h @@ -67,6 +67,11 @@ typedef struct PyMemberDef { PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, struct PyMemberDef *); PyAPI_FUNC(int) PyMember_SetOne(char *, struct PyMemberDef *, PyObject *); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(PyObject *) _PyMemberDescr_GetOne(const char *, PyMemberDescrObject *); +PyAPI_FUNC(int) _PyMemberDescr_SetOne(char *, PyMemberDescrObject *, PyObject *); +#endif + #ifdef __cplusplus } diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 5c70037f39a2db..1232d306eba3bf 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -86,39 +86,39 @@ def main(): 8 0.312 0.039 0.400 0.050 profilee.py:88(helper2) 8 0.064 0.008 0.080 0.010 profilee.py:98(subhelper)""" _ProfileOutput['print_callers'] = """\ -profilee.py:110(__getattr__) <- 16 0.016 0.016 profilee.py:98(subhelper) -profilee.py:25(testfunc) <- 1 0.270 1.000 :1() -profilee.py:35(factorial) <- 1 0.014 0.130 profilee.py:25(testfunc) - 20/3 0.130 0.147 profilee.py:35(factorial) - 2 0.006 0.040 profilee.py:84(helper2_indirect) -profilee.py:48(mul) <- 20 0.020 0.020 profilee.py:35(factorial) -profilee.py:55(helper) <- 2 0.040 0.600 profilee.py:25(testfunc) -profilee.py:73(helper1) <- 4 0.116 0.120 profilee.py:55(helper) -profilee.py:84(helper2_indirect) <- 2 0.000 0.140 profilee.py:55(helper) -profilee.py:88(helper2) <- 6 0.234 0.300 profilee.py:55(helper) - 2 0.078 0.100 profilee.py:84(helper2_indirect) -profilee.py:98(subhelper) <- 8 0.064 0.080 profilee.py:88(helper2) -{built-in method builtins.hasattr} <- 4 0.000 0.004 profilee.py:73(helper1) - 8 0.000 0.008 profilee.py:88(helper2) -{built-in method sys.exc_info} <- 4 0.000 0.000 profilee.py:73(helper1) -{method 'append' of 'list' objects} <- 4 0.000 0.000 profilee.py:73(helper1)""" +profilee.py:110(__getattr__) <- 16 0.016 0.016 profilee.py:98(subhelper) +profilee.py:25(testfunc) <- 1 0.270 1.000 :1() +profilee.py:35(factorial) <- 1 0.014 0.130 profilee.py:25(testfunc) + 20/3 0.130 0.147 profilee.py:35(factorial) + 2 0.006 0.040 profilee.py:84(helper2_indirect) +profilee.py:48(mul) <- 20 0.020 0.020 profilee.py:35(factorial) +profilee.py:55(helper) <- 2 0.040 0.600 profilee.py:25(testfunc) +profilee.py:73(helper1) <- 4 0.116 0.120 profilee.py:55(helper) +profilee.py:84(helper2_indirect) <- 2 0.000 0.140 profilee.py:55(helper) +profilee.py:88(helper2) <- 6 0.234 0.300 profilee.py:55(helper) + 2 0.078 0.100 profilee.py:84(helper2_indirect) +profilee.py:98(subhelper) <- 8 0.064 0.080 profilee.py:88(helper2) +{built-in method builtins.hasattr} <- 4 0.000 0.004 profilee.py:73(helper1) + 8 0.000 0.008 profilee.py:88(helper2) +{built-in method sys.exc_info} <- 4 0.000 0.000 profilee.py:73(helper1) +{method 'append' of 'list' objects} <- 4 0.000 0.000 profilee.py:73(helper1)""" _ProfileOutput['print_callees'] = """\ -:1() -> 1 0.270 1.000 profilee.py:25(testfunc) -profilee.py:110(__getattr__) -> -profilee.py:25(testfunc) -> 1 0.014 0.130 profilee.py:35(factorial) - 2 0.040 0.600 profilee.py:55(helper) -profilee.py:35(factorial) -> 20/3 0.130 0.147 profilee.py:35(factorial) - 20 0.020 0.020 profilee.py:48(mul) -profilee.py:48(mul) -> -profilee.py:55(helper) -> 4 0.116 0.120 profilee.py:73(helper1) - 2 0.000 0.140 profilee.py:84(helper2_indirect) - 6 0.234 0.300 profilee.py:88(helper2) -profilee.py:73(helper1) -> 4 0.000 0.004 {built-in method builtins.hasattr} -profilee.py:84(helper2_indirect) -> 2 0.006 0.040 profilee.py:35(factorial) - 2 0.078 0.100 profilee.py:88(helper2) -profilee.py:88(helper2) -> 8 0.064 0.080 profilee.py:98(subhelper) -profilee.py:98(subhelper) -> 16 0.016 0.016 profilee.py:110(__getattr__) -{built-in method builtins.hasattr} -> 12 0.012 0.012 profilee.py:110(__getattr__)""" +:1() -> 1 0.270 1.000 profilee.py:25(testfunc) +profilee.py:110(__getattr__) -> +profilee.py:25(testfunc) -> 1 0.014 0.130 profilee.py:35(factorial) + 2 0.040 0.600 profilee.py:55(helper) +profilee.py:35(factorial) -> 20/3 0.130 0.147 profilee.py:35(factorial) + 20 0.020 0.020 profilee.py:48(mul) +profilee.py:48(mul) -> +profilee.py:55(helper) -> 4 0.116 0.120 profilee.py:73(helper1) + 2 0.000 0.140 profilee.py:84(helper2_indirect) + 6 0.234 0.300 profilee.py:88(helper2) +profilee.py:73(helper1) -> 4 0.000 0.004 {built-in method builtins.hasattr} +profilee.py:84(helper2_indirect) -> 2 0.006 0.040 profilee.py:35(factorial) + 2 0.078 0.100 profilee.py:88(helper2) +profilee.py:88(helper2) -> 8 0.064 0.080 profilee.py:98(subhelper) +profilee.py:98(subhelper) -> 16 0.016 0.016 profilee.py:110(__getattr__) +{built-in method builtins.hasattr} -> 12 0.012 0.012 profilee.py:110(__getattr__)""" if __name__ == "__main__": main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 49f2722d95140e..47b54284d91780 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1080,7 +1080,7 @@ def test_objecttypes(self): # buffer # XXX # builtin_function_or_method - check(len, size('5P')) + check(len, size('8Pi')) # bytearray samples = [b'', b'u'*100000] for sample in samples: @@ -1111,15 +1111,15 @@ def inner(): # complex check(complex(0,1), size('2d')) # method_descriptor (descriptor object) - check(str.lower, size('3PPP')) + check(str.lower, size('3P4PiP')) # classmethod_descriptor (descriptor object) # XXX # member_descriptor (descriptor object) import datetime - check(datetime.timedelta.days, size('3PP')) + check(datetime.timedelta.days, size('3PiniP')) # getset_descriptor (descriptor object) import collections - check(collections.defaultdict.default_factory, size('3PP')) + check(collections.defaultdict.default_factory, size('3P4P')) # wrapper_descriptor (descriptor object) check(int.__add__, size('3P2P')) # method-wrapper (descriptor object) diff --git a/Misc/NEWS.d/next/C API/2019-06-18-00-55-12.bpo-37270.2z98X7.rst b/Misc/NEWS.d/next/C API/2019-06-18-00-55-12.bpo-37270.2z98X7.rst new file mode 100644 index 00000000000000..b4ba25a4e0e9f0 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-06-18-00-55-12.bpo-37270.2z98X7.rst @@ -0,0 +1,6 @@ +Ensure that all of the memory needed by a type created with +:c:func:`PyType_FromSpec` or :c:func:`PyType_FromSpecWithBases` is managed by +Python. Previously, the memory referred to by ``Py_tp_methods``, +``Py_tp_getset``, ``Py_tp_members`` and the various ``char*`` fields they +contained needed to outlive the type. Now, these arrays and strings are copied +and managed by the type itself. \ No newline at end of file diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index c5a6f4445872ca..6a370fc3ad9793 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -132,42 +132,40 @@ normalizeUserObj(PyObject *obj) if (modname != NULL) { if (!_PyUnicode_EqualToASCIIString(modname, "builtins")) { PyObject *result; - result = PyUnicode_FromFormat("<%U.%s>", modname, - fn->m_ml->ml_name); + result = PyUnicode_FromFormat("<%U.%S>", modname, + fn->m_base.name); Py_DECREF(modname); return result; } Py_DECREF(modname); } - return PyUnicode_FromFormat("<%s>", fn->m_ml->ml_name); + return PyUnicode_FromFormat("<%S>", fn->m_base.name); } else { /* built-in method: try to return repr(getattr(type(__self__), __name__)) */ PyObject *self = fn->m_self; - PyObject *name = PyUnicode_FromString(fn->m_ml->ml_name); + PyObject *name = fn->m_base.name; PyObject *modname = fn->m_module; - if (name != NULL) { - PyObject *mo = _PyType_Lookup(Py_TYPE(self), name); - Py_XINCREF(mo); - Py_DECREF(name); - if (mo != NULL) { - PyObject *res = PyObject_Repr(mo); - Py_DECREF(mo); - if (res != NULL) - return res; - } + PyObject *mo = _PyType_Lookup(Py_TYPE(self), name); + Py_XINCREF(mo); + if (mo != NULL) { + PyObject *res = PyObject_Repr(mo); + Py_DECREF(mo); + if (res != NULL) + return res; } + /* Otherwise, use __module__ */ PyErr_Clear(); if (modname != NULL && PyUnicode_Check(modname)) - return PyUnicode_FromFormat("", - modname, fn->m_ml->ml_name); + return PyUnicode_FromFormat("", + modname, fn->m_base.name); else - return PyUnicode_FromFormat("", - fn->m_ml->ml_name); + return PyUnicode_FromFormat("", + fn->m_base.name); } } @@ -409,7 +407,7 @@ profiler_callback(PyObject *self, PyFrameObject *frame, int what, if ((((ProfilerObject *)self)->flags & POF_BUILTINS) && PyCFunction_Check(arg)) { ptrace_enter_call(self, - ((PyCFunctionObject *)arg)->m_ml, + &((PyCFunctionObject *)arg)->m_base, arg); } break; @@ -421,7 +419,7 @@ profiler_callback(PyObject *self, PyFrameObject *frame, int what, if ((((ProfilerObject *)self)->flags & POF_BUILTINS) && PyCFunction_Check(arg)) { ptrace_leave_call(self, - ((PyCFunctionObject *)arg)->m_ml); + &((PyCFunctionObject *)arg)->m_base); } break; diff --git a/Objects/abstract.c b/Objects/abstract.c index 77d09143aa0764..c66c1624319b9e 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -5,6 +5,7 @@ #include #include "structmember.h" /* we need the offsetof() macro from there */ #include "longintrepr.h" +#include "bltinmodule.h" @@ -840,7 +841,7 @@ binary_op(PyObject *v, PyObject *w, const int op_slot, const char *op_name) if (op_slot == NB_SLOT(nb_rshift) && PyCFunction_Check(v) && - strcmp(((PyCFunctionObject *)v)->m_ml->ml_name, "print") == 0) + PyCFunction_GET_FUNCTION(v) == (PyCFunction)_PyBuiltin_Print) { PyErr_Format(PyExc_TypeError, "unsupported operand type(s) for %.100s: " diff --git a/Objects/call.c b/Objects/call.c index f9a3207e962289..e089e4f85be5d5 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -453,22 +453,23 @@ _PyFunction_Vectorcall(PyObject *func, PyObject* const* stack, /* --- PyCFunction call functions --------------------------------- */ PyObject * -_PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs) +_PyCFunctionBase_RawFastCallDict(PyCFunctionBase *func, + PyObject *self, + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs) { /* _PyMethodDef_RawFastCallDict() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ assert(!PyErr_Occurred()); - assert(method != NULL); + assert(func != NULL); assert(nargs >= 0); assert(nargs == 0 || args != NULL); assert(kwargs == NULL || PyDict_Check(kwargs)); - PyCFunction meth = method->ml_meth; - int flags = method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); + PyCFunction meth = func->meth; + int flags = func->flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); PyObject *result = NULL; if (Py_EnterRecursiveCall(" while calling a Python object")) { @@ -484,8 +485,8 @@ _PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, if (nargs != 0) { PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%zd given)", - method->ml_name, nargs); + "%S() takes no arguments (%zd given)", + func->name, nargs); goto exit; } @@ -499,8 +500,8 @@ _PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, if (nargs != 1) { PyErr_Format(PyExc_TypeError, - "%.200s() takes exactly one argument (%zd given)", - method->ml_name, nargs); + "%S() takes exactly one argument (%zd given)", + func->name, nargs); goto exit; } @@ -565,7 +566,7 @@ _PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, default: PyErr_SetString(PyExc_SystemError, - "Bad call flags in _PyMethodDef_RawFastCallDict. " + "Bad call flags in _PyCFunctionBase_RawFastCallDict. " "METH_OLDARGS is no longer supported!"); goto exit; } @@ -574,15 +575,14 @@ _PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, no_keyword_error: PyErr_Format(PyExc_TypeError, - "%.200s() takes no keyword arguments", - method->ml_name); + "%S() takes no keyword arguments", + func->name); exit: Py_LeaveRecursiveCall(); return result; } - PyObject * _PyCFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, @@ -593,32 +593,32 @@ _PyCFunction_FastCallDict(PyObject *func, assert(func != NULL); assert(PyCFunction_Check(func)); - result = _PyMethodDef_RawFastCallDict(((PyCFunctionObject*)func)->m_ml, - PyCFunction_GET_SELF(func), - args, nargs, kwargs); + result = _PyCFunctionBase_RawFastCallDict(&((PyCFunctionObject *)func)->m_base, + PyCFunction_GET_SELF(func), + args, nargs, kwargs); result = _Py_CheckFunctionResult(func, result, NULL); return result; } - PyObject * -_PyMethodDef_RawFastCallKeywords(PyMethodDef *method, PyObject *self, - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwnames) +_PyCFunctionBase_RawFastCallKeywords(PyCFunctionBase *func, + PyObject* self, + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwnames) { /* _PyMethodDef_RawFastCallKeywords() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ assert(!PyErr_Occurred()); - assert(method != NULL); + assert(func != NULL); assert(nargs >= 0); assert(kwnames == NULL || PyTuple_CheckExact(kwnames)); /* kwnames must only contains str strings, no subclass, and all keys must be unique */ - PyCFunction meth = method->ml_meth; - int flags = method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); + PyCFunction meth = func->meth; + int flags = func->flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); Py_ssize_t nkwargs = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); PyObject *result = NULL; @@ -635,8 +635,8 @@ _PyMethodDef_RawFastCallKeywords(PyMethodDef *method, PyObject *self, if (nargs != 0) { PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%zd given)", - method->ml_name, nargs); + "%S() takes no arguments (%zd given)", + func->name, nargs); goto exit; } @@ -650,8 +650,8 @@ _PyMethodDef_RawFastCallKeywords(PyMethodDef *method, PyObject *self, if (nargs != 1) { PyErr_Format(PyExc_TypeError, - "%.200s() takes exactly one argument (%zd given)", - method->ml_name, nargs); + "%S() takes exactly one argument (%zd given)", + func->name, nargs); goto exit; } @@ -713,7 +713,7 @@ _PyMethodDef_RawFastCallKeywords(PyMethodDef *method, PyObject *self, default: PyErr_SetString(PyExc_SystemError, - "Bad call flags in _PyMethodDef_RawFastCallKeywords. " + "Bad call flags in _PyCFunctionBase_RawFastCallKeywords. " "METH_OLDARGS is no longer supported!"); goto exit; } @@ -722,15 +722,14 @@ _PyMethodDef_RawFastCallKeywords(PyMethodDef *method, PyObject *self, no_keyword_error: PyErr_Format(PyExc_TypeError, - "%.200s() takes no keyword arguments", - method->ml_name); + "%S() takes no keyword arguments", + func->name); exit: Py_LeaveRecursiveCall(); return result; } - PyObject * _PyCFunction_Vectorcall(PyObject *func, PyObject *const *args, size_t nargsf, @@ -742,9 +741,9 @@ _PyCFunction_Vectorcall(PyObject *func, assert(PyCFunction_Check(func)); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - result = _PyMethodDef_RawFastCallKeywords(((PyCFunctionObject*)func)->m_ml, - PyCFunction_GET_SELF(func), - args, nargs, kwnames); + result = _PyCFunctionBase_RawFastCallKeywords(&((PyCFunctionObject*)func)->m_base, + PyCFunction_GET_SELF(func), + args, nargs, kwnames); result = _Py_CheckFunctionResult(func, result, NULL); return result; } @@ -772,8 +771,8 @@ cfunction_call_varargs(PyObject *func, PyObject *args, PyObject *kwargs) } else { if (kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) { - PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", - ((PyCFunctionObject*)func)->m_ml->ml_name); + PyErr_Format(PyExc_TypeError, "%S() takes no keyword arguments", + ((PyCFunctionObject*)func)->m_base.name); return NULL; } diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 806c0af97e24b6..385daf30efba67 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -22,6 +22,27 @@ descr_dealloc(PyDescrObject *descr) PyObject_GC_Del(descr); } +static void +method_dealloc(PyMethodDescrObject *meth) +{ + _PyCFunctionBase_Clear(&meth->d_base); + descr_dealloc((PyDescrObject *)meth); +} + +static void +member_dealloc(PyMemberDescrObject *getset) +{ + Py_XDECREF(getset->d_doc); + descr_dealloc((PyDescrObject *)getset); +} + +static void +getset_dealloc(PyGetSetDescrObject *getset) +{ + Py_XDECREF(getset->d_doc); + descr_dealloc((PyDescrObject *)getset); +} + static PyObject * descr_name(PyDescrObject *descr) { @@ -124,7 +145,8 @@ classmethod_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) ((PyTypeObject *)type)->tp_name); return NULL; } - return PyCFunction_NewEx(descr->d_method, type, NULL); + + return _PyCFunction_NewFromBase(&descr->d_base, type, NULL); } static PyObject * @@ -134,7 +156,8 @@ method_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type) if (descr_check((PyDescrObject *)descr, obj, &res)) return res; - return PyCFunction_NewEx(descr->d_method, obj, NULL); + + return _PyCFunction_NewFromBase(&descr->d_base, obj, NULL); } static PyObject * @@ -145,14 +168,14 @@ member_get(PyMemberDescrObject *descr, PyObject *obj, PyObject *type) if (descr_check((PyDescrObject *)descr, obj, &res)) return res; - if (descr->d_member->flags & READ_RESTRICTED) { - if (PySys_Audit("object.__getattr__", "Os", - obj ? obj : Py_None, descr->d_member->name) < 0) { + if (descr->d_flags & READ_RESTRICTED) { + if (PySys_Audit("object.__getattr__", "OS", + obj ? obj : Py_None, descr->d_common.d_name) < 0) { return NULL; } } - return PyMember_GetOne((char *)obj, descr->d_member); + return _PyMemberDescr_GetOne((char *)obj, descr); } static PyObject * @@ -162,8 +185,8 @@ getset_get(PyGetSetDescrObject *descr, PyObject *obj, PyObject *type) if (descr_check((PyDescrObject *)descr, obj, &res)) return res; - if (descr->d_getset->get != NULL) - return descr->d_getset->get(obj, descr->d_getset->closure); + if (descr->d_get != NULL) + return descr->d_get(obj, descr->d_closure); PyErr_Format(PyExc_AttributeError, "attribute '%V' of '%.100s' objects is not readable", descr_name((PyDescrObject *)descr), "?", @@ -206,7 +229,7 @@ member_set(PyMemberDescrObject *descr, PyObject *obj, PyObject *value) if (descr_setcheck((PyDescrObject *)descr, obj, value, &res)) return res; - return PyMember_SetOne((char *)obj, descr->d_member, value); + return _PyMemberDescr_SetOne((char *)obj, descr, value); } static int @@ -216,9 +239,8 @@ getset_set(PyGetSetDescrObject *descr, PyObject *obj, PyObject *value) if (descr_setcheck((PyDescrObject *)descr, obj, value, &res)) return res; - if (descr->d_getset->set != NULL) - return descr->d_getset->set(obj, value, - descr->d_getset->closure); + if (descr->d_set != NULL) + return descr->d_set(obj, value, descr->d_closure); PyErr_Format(PyExc_AttributeError, "attribute '%V' of '%.100s' objects is not writable", descr_name((PyDescrObject *)descr), "?", @@ -255,9 +277,9 @@ methoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyObject *kwargs) return NULL; } - result = _PyMethodDef_RawFastCallDict(descr->d_method, self, - &_PyTuple_ITEMS(args)[1], nargs - 1, - kwargs); + result = _PyCFunctionBase_RawFastCallDict(&descr->d_base, self, + &_PyTuple_ITEMS(args)[1], nargs - 1, + kwargs); result = _Py_CheckFunctionResult((PyObject *)descr, result, NULL); return result; } @@ -294,8 +316,8 @@ _PyMethodDescr_Vectorcall(PyObject *descrobj, return NULL; } - result = _PyMethodDef_RawFastCallKeywords(descr->d_method, self, - args+1, nargs-1, kwnames); + result = _PyCFunctionBase_RawFastCallKeywords(&descr->d_base, self, + args+1, nargs-1, kwnames); result = _Py_CheckFunctionResult((PyObject *)descr, result, NULL); return result; } @@ -395,13 +417,15 @@ wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) static PyObject * method_get_doc(PyMethodDescrObject *descr, void *closure) { - return _PyType_GetDocFromInternalDoc(descr->d_method->ml_name, descr->d_method->ml_doc); + Py_INCREF(descr->d_base.doc); + return descr->d_base.doc; } static PyObject * method_get_text_signature(PyMethodDescrObject *descr, void *closure) { - return _PyType_GetTextSignatureFromInternalDoc(descr->d_method->ml_name, descr->d_method->ml_doc); + Py_INCREF(descr->d_base.signature); + return descr->d_base.signature; } static PyObject * @@ -471,10 +495,8 @@ static PyGetSetDef method_getset[] = { static PyObject * member_get_doc(PyMemberDescrObject *descr, void *closure) { - if (descr->d_member->doc == NULL) { - Py_RETURN_NONE; - } - return PyUnicode_FromString(descr->d_member->doc); + Py_INCREF(descr->d_doc); + return descr->d_doc; } static PyGetSetDef member_getset[] = { @@ -486,10 +508,8 @@ static PyGetSetDef member_getset[] = { static PyObject * getset_get_doc(PyGetSetDescrObject *descr, void *closure) { - if (descr->d_getset->doc == NULL) { - Py_RETURN_NONE; - } - return PyUnicode_FromString(descr->d_getset->doc); + Py_INCREF(descr->d_doc); + return descr->d_doc; } static PyGetSetDef getset_getset[] = { @@ -570,7 +590,7 @@ PyTypeObject PyClassMethodDescr_Type = { "classmethod_descriptor", sizeof(PyMethodDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + (destructor)method_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -607,7 +627,7 @@ PyTypeObject PyMemberDescr_Type = { "member_descriptor", sizeof(PyMemberDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + (destructor)member_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -644,7 +664,7 @@ PyTypeObject PyGetSetDescr_Type = { "getset_descriptor", sizeof(PyGetSetDescrObject), 0, - (destructor)descr_dealloc, /* tp_dealloc */ + (destructor)getset_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -742,10 +762,11 @@ PyDescr_NewMethod(PyTypeObject *type, PyMethodDef *method) descr = (PyMethodDescrObject *)descr_new(&PyMethodDescr_Type, type, method->ml_name); - if (descr != NULL) { - descr->d_method = method; - descr->vectorcall = _PyMethodDescr_Vectorcall; + if (descr != NULL && _PyCFunctionBase_FromMethodDef(&descr->d_base, method)) { + Py_DECREF(descr); + return NULL; } + descr->vectorcall = _PyMethodDescr_Vectorcall; return (PyObject *)descr; } @@ -756,8 +777,10 @@ PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) descr = (PyMethodDescrObject *)descr_new(&PyClassMethodDescr_Type, type, method->ml_name); - if (descr != NULL) - descr->d_method = method; + if (descr != NULL && _PyCFunctionBase_FromMethodDef(&descr->d_base, method)) { + Py_DECREF(descr); + return NULL; + } return (PyObject *)descr; } @@ -768,8 +791,23 @@ PyDescr_NewMember(PyTypeObject *type, PyMemberDef *member) descr = (PyMemberDescrObject *)descr_new(&PyMemberDescr_Type, type, member->name); - if (descr != NULL) - descr->d_member = member; + if (descr != NULL) { + if (member->doc == NULL) { + Py_INCREF(Py_None); + descr->d_doc = Py_None; + } + else { + descr->d_doc = PyUnicode_InternFromString(member->doc); + if (descr->d_doc == NULL) { + Py_DECREF(descr->d_common.d_name); + return NULL; + } + } + + descr->d_member_type = member->type; + descr->d_offset = member->offset; + descr->d_flags = member->flags; + } return (PyObject *)descr; } @@ -780,8 +818,23 @@ PyDescr_NewGetSet(PyTypeObject *type, PyGetSetDef *getset) descr = (PyGetSetDescrObject *)descr_new(&PyGetSetDescr_Type, type, getset->name); - if (descr != NULL) - descr->d_getset = getset; + if (descr != NULL) { + if (getset->doc == NULL) { + Py_INCREF(Py_None); + descr->d_doc = Py_None; + } + else { + descr->d_doc = PyUnicode_InternFromString(getset->doc); + if (descr->d_doc == NULL) { + Py_DECREF(descr->d_common.d_name); + return NULL; + } + } + + descr->d_get = getset->get; + descr->d_set = getset->set; + descr->d_closure = getset->closure; + } return (PyObject *)descr; } diff --git a/Objects/methodobject.c b/Objects/methodobject.c index c3bc0184796e40..68c81438cf1ce6 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -16,6 +16,49 @@ static int numfree = 0; #define PyCFunction_MAXFREELIST 256 #endif +int +_PyCFunctionBase_FromMethodDef(PyCFunctionBase *base, PyMethodDef *ml) +{ + base->name = PyUnicode_InternFromString(ml->ml_name); + if (base->name == NULL) { + goto error; + } + if (ml->ml_doc == NULL) { + Py_INCREF(Py_None); + base->doc = Py_None; + + Py_INCREF(Py_None); + base->signature = Py_None; + } + else { + base->doc = _PyType_GetDocFromInternalDoc(ml->ml_name, ml->ml_doc); + if (base->doc == NULL) { + goto error; + } + + base->signature = _PyType_GetTextSignatureFromInternalDoc(ml->ml_name, + ml->ml_doc); + if (base->signature == NULL) { + goto error; + } + } + base->meth = ml->ml_meth; + base->flags = ml->ml_flags; + + return 0; + +error: + _PyCFunctionBase_Clear(base); + return -1; +} + +void _PyCFunctionBase_Clear(PyCFunctionBase *base) +{ + Py_CLEAR(base->name); + Py_CLEAR(base->doc); + Py_CLEAR(base->signature); +} + /* undefine macro trampoline to PyCFunction_NewEx */ #undef PyCFunction_New @@ -28,6 +71,19 @@ PyCFunction_New(PyMethodDef *ml, PyObject *self) PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) { + PyCFunctionBase base = {0}; + + if (_PyCFunctionBase_FromMethodDef(&base, ml)) { + return NULL; + } + PyObject *out = _PyCFunction_NewFromBase(&base, self, module); + + _PyCFunctionBase_Clear(&base); + return out; +} + +PyObject * +_PyCFunction_NewFromBase(PyCFunctionBase *base, PyObject *self, PyObject *module) { PyCFunctionObject *op; op = free_list; if (op != NULL) { @@ -41,12 +97,14 @@ PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) return NULL; } op->m_weakreflist = NULL; - op->m_ml = ml; - Py_XINCREF(self); - op->m_self = self; - Py_XINCREF(module); - op->m_module = module; - if (ml->ml_flags & METH_VARARGS) { + + /* Take ownership of the objects on the given base. */ + Py_INCREF(base->name); + Py_INCREF(base->doc); + Py_INCREF(base->signature); + op->m_base = *base; + + if (base->flags & METH_VARARGS) { /* For METH_VARARGS functions, it's more efficient to use tp_call * instead of vectorcall. */ op->vectorcall = NULL; @@ -54,6 +112,12 @@ PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) else { op->vectorcall = _PyCFunction_Vectorcall; } + + Py_XINCREF(self); + op->m_self = self; + Py_XINCREF(module); + op->m_module = module; + _PyObject_GC_TRACK(op); return (PyObject *)op; } @@ -97,6 +161,7 @@ meth_dealloc(PyCFunctionObject *m) if (m->m_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*) m); } + _PyCFunctionBase_Clear(&m->m_base); Py_XDECREF(m->m_self); Py_XDECREF(m->m_module); if (numfree < PyCFunction_MAXFREELIST) { @@ -114,11 +179,13 @@ meth_reduce(PyCFunctionObject *m, PyObject *Py_UNUSED(ignored)) { _Py_IDENTIFIER(getattr); - if (m->m_self == NULL || PyModule_Check(m->m_self)) - return PyUnicode_FromString(m->m_ml->ml_name); + if (m->m_self == NULL || PyModule_Check(m->m_self)) { + Py_INCREF(m->m_base.name); + return m->m_base.name; + } - return Py_BuildValue("N(Os)", _PyEval_GetBuiltinId(&PyId_getattr), - m->m_self, m->m_ml->ml_name); + return Py_BuildValue("N(OS)", _PyEval_GetBuiltinId(&PyId_getattr), + m->m_self, m->m_base.name); } static PyMethodDef meth_methods[] = { @@ -126,24 +193,6 @@ static PyMethodDef meth_methods[] = { {NULL, NULL} }; -static PyObject * -meth_get__text_signature__(PyCFunctionObject *m, void *closure) -{ - return _PyType_GetTextSignatureFromInternalDoc(m->m_ml->ml_name, m->m_ml->ml_doc); -} - -static PyObject * -meth_get__doc__(PyCFunctionObject *m, void *closure) -{ - return _PyType_GetDocFromInternalDoc(m->m_ml->ml_name, m->m_ml->ml_doc); -} - -static PyObject * -meth_get__name__(PyCFunctionObject *m, void *closure) -{ - return PyUnicode_FromString(m->m_ml->ml_name); -} - static PyObject * meth_get__qualname__(PyCFunctionObject *m, void *closure) { @@ -158,8 +207,10 @@ meth_get__qualname__(PyCFunctionObject *m, void *closure) PyObject *type, *type_qualname, *res; _Py_IDENTIFIER(__qualname__); - if (m->m_self == NULL || PyModule_Check(m->m_self)) - return PyUnicode_FromString(m->m_ml->ml_name); + if (m->m_self == NULL || PyModule_Check(m->m_self)) { + Py_INCREF(m->m_base.name); + return m->m_base.name; + } type = PyType_Check(m->m_self) ? m->m_self : (PyObject*)Py_TYPE(m->m_self); @@ -174,7 +225,7 @@ meth_get__qualname__(PyCFunctionObject *m, void *closure) return NULL; } - res = PyUnicode_FromFormat("%S.%s", type_qualname, m->m_ml->ml_name); + res = PyUnicode_FromFormat("%S.%S", type_qualname, m->m_base.name); Py_DECREF(type_qualname); return res; } @@ -200,18 +251,18 @@ meth_get__self__(PyCFunctionObject *m, void *closure) } static PyGetSetDef meth_getsets [] = { - {"__doc__", (getter)meth_get__doc__, NULL, NULL}, - {"__name__", (getter)meth_get__name__, NULL, NULL}, {"__qualname__", (getter)meth_get__qualname__, NULL, NULL}, {"__self__", (getter)meth_get__self__, NULL, NULL}, - {"__text_signature__", (getter)meth_get__text_signature__, NULL, NULL}, {0} }; #define OFF(x) offsetof(PyCFunctionObject, x) static PyMemberDef meth_members[] = { - {"__module__", T_OBJECT, OFF(m_module), PY_WRITE_RESTRICTED}, + {"__module__", T_OBJECT, OFF(m_module), PY_WRITE_RESTRICTED}, + {"__name__", T_OBJECT, OFF(m_base.name), READONLY}, + {"__doc__", T_OBJECT, OFF(m_base.doc), READONLY}, + {"__text_signature__", T_OBJECT, OFF(m_base.signature), READONLY}, {NULL} }; @@ -219,10 +270,10 @@ static PyObject * meth_repr(PyCFunctionObject *m) { if (m->m_self == NULL || PyModule_Check(m->m_self)) - return PyUnicode_FromFormat("", - m->m_ml->ml_name); - return PyUnicode_FromFormat("", - m->m_ml->ml_name, + return PyUnicode_FromFormat("", + m->m_base.name); + return PyUnicode_FromFormat("", + m->m_base.name, m->m_self->ob_type->tp_name, m->m_self); } @@ -244,7 +295,7 @@ meth_richcompare(PyObject *self, PyObject *other, int op) b = (PyCFunctionObject *)other; eq = a->m_self == b->m_self; if (eq) - eq = a->m_ml->ml_meth == b->m_ml->ml_meth; + eq = PyCFunction_GET_FUNCTION(a) == PyCFunction_GET_FUNCTION(b); if (op == Py_EQ) res = eq ? Py_True : Py_False; else @@ -258,7 +309,7 @@ meth_hash(PyCFunctionObject *a) { Py_hash_t x, y; x = _Py_HashPointer(a->m_self); - y = _Py_HashPointer((void*)(a->m_ml->ml_meth)); + y = _Py_HashPointer((void*)PyCFunction_GET_FUNCTION(a)); x ^= y; if (x == -1) x = -2; diff --git a/Objects/structseq.c b/Objects/structseq.c index 2c25e1646a2a67..8305ef721d9822 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -110,7 +110,7 @@ structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict) { PyObject *ob; PyStructSequence *res = NULL; - Py_ssize_t len, min_len, max_len, i, n_unnamed_fields; + Py_ssize_t len, min_len, max_len, i; arg = PySequence_Fast(arg, "constructor requires a sequence"); @@ -129,7 +129,6 @@ structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict) len = PySequence_Fast_GET_SIZE(arg); min_len = VISIBLE_SIZE_TP(type); max_len = REAL_SIZE_TP(type); - n_unnamed_fields = UNNAMED_FIELDS_TP(type); if (min_len != max_len) { if (len < min_len) { @@ -168,15 +167,40 @@ structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict) Py_INCREF(v); res->ob_item[i] = v; } - for (; i < max_len; ++i) { - if (dict && (ob = PyDict_GetItemString( - dict, type->tp_members[i-n_unnamed_fields].name))) { - } - else { - ob = Py_None; + + if (dict != NULL) { + Py_ssize_t field = 0; + PyObject *key; + PyObject *value; + Py_ssize_t ix = 0; + while (PyDict_Next(type->tp_dict, &ix, &key, &value)) { + if (Py_TYPE(value) != &PyMemberDescr_Type) { + continue; + } + PyMemberDescrObject *descr = (PyMemberDescrObject *)value; + if (field < len) { + ++field; + continue; + } + + ob = PyDict_GetItem(dict, descr->d_common.d_name); + if (ob == NULL) { + ob = Py_None; + } + + /* Don't go through _PyMemberDescr_SetOne because we might need + to initialize a read-only member. */ + Py_INCREF(ob); + *(PyObject **)((char *)res + descr->d_offset) = ob; + + ++field; + ++len; } - Py_INCREF(ob); - res->ob_item[i] = ob; + } + + for (i = len; i < max_len; ++i) { + Py_INCREF(Py_None); + res->ob_item[i] = Py_None; } Py_DECREF(arg); @@ -188,13 +212,27 @@ structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict) static PyObject * structseq_repr(PyStructSequence *obj) { - PyTypeObject *typ = Py_TYPE(obj); _PyUnicodeWriter writer; + PyObject *type_name; + PyTypeObject *typ = Py_TYPE(obj); - /* Write "typename(" */ - PyObject *type_name = PyUnicode_DecodeUTF8(typ->tp_name, - strlen(typ->tp_name), - NULL); + if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HEAPTYPE) { + _Py_IDENTIFIER(__module__); + + PyObject *mod = _PyObject_GetAttrId((PyObject *)typ, &PyId___module__); + if (mod == NULL) { + PyErr_SetObject(PyExc_KeyError, PyId___module__.object); + return NULL; + } + type_name = PyUnicode_FromFormat("%S.%S", + mod, ((PyHeapTypeObject *)typ)->ht_name); + Py_DECREF(mod); + } + else { + type_name = PyUnicode_DecodeUTF8(typ->tp_name, + strlen(typ->tp_name), + NULL); + } if (type_name == NULL) { return NULL; } @@ -209,14 +247,27 @@ structseq_repr(PyStructSequence *obj) Py_DECREF(type_name); goto error; } - Py_DECREF(type_name); + /* Write "typename(" */ if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) { goto error; } - for (Py_ssize_t i=0; i < VISIBLE_SIZE(obj); i++) { - if (i > 0) { + Py_ssize_t fields_written = 0; + PyObject *key; + PyObject *value; + Py_ssize_t ix = 0; + while (PyDict_Next(typ->tp_dict, &ix, &key, &value)) { + if (Py_TYPE(value) != &PyMemberDescr_Type) { + continue; + } + if (fields_written >= VISIBLE_SIZE(obj)) { + break; + } + + PyMemberDescrObject *descr = (PyMemberDescrObject *)value; + + if (fields_written > 0) { /* Write ", " */ if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { goto error; @@ -224,28 +275,17 @@ structseq_repr(PyStructSequence *obj) } /* Write "name=repr" */ - const char *name_utf8 = typ->tp_members[i].name; - if (name_utf8 == NULL) { - PyErr_Format(PyExc_SystemError, "In structseq_repr(), member %zd name is NULL" - " for type %.500s", i, typ->tp_name); - goto error; - } - - PyObject *name = PyUnicode_DecodeUTF8(name_utf8, strlen(name_utf8), NULL); - if (name == NULL) { - goto error; - } + PyObject *name = descr->d_common.d_name; if (_PyUnicodeWriter_WriteStr(&writer, name) < 0) { Py_DECREF(name); goto error; } - Py_DECREF(name); if (_PyUnicodeWriter_WriteChar(&writer, '=') < 0) { goto error; } - PyObject *value = PyStructSequence_GET_ITEM(obj, i); + PyObject *value = _PyMemberDescr_GetOne((char *)obj, descr); assert(value != NULL); PyObject *repr = PyObject_Repr(value); if (repr == NULL) { @@ -256,15 +296,22 @@ structseq_repr(PyStructSequence *obj) goto error; } Py_DECREF(repr); + + ++fields_written; + } + if (PyErr_Occurred()) { + goto error; } if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) { goto error; } + Py_DECREF(type_name); return _PyUnicodeWriter_Finish(&writer); error: + Py_DECREF(type_name); _PyUnicodeWriter_Dealloc(&writer); return NULL; } @@ -276,11 +323,9 @@ structseq_reduce(PyStructSequence* self, PyObject *Py_UNUSED(ignored)) PyObject* tup = NULL; PyObject* dict = NULL; PyObject* result; - Py_ssize_t n_fields, n_visible_fields, n_unnamed_fields, i; + Py_ssize_t n_visible_fields; - n_fields = REAL_SIZE(self); n_visible_fields = VISIBLE_SIZE(self); - n_unnamed_fields = UNNAMED_FIELDS(self); tup = _PyTuple_FromArray(self->ob_item, n_visible_fields); if (!tup) goto error; @@ -289,10 +334,25 @@ structseq_reduce(PyStructSequence* self, PyObject *Py_UNUSED(ignored)) if (!dict) goto error; - for (i = n_visible_fields; i < n_fields; i++) { - const char *n = Py_TYPE(self)->tp_members[i-n_unnamed_fields].name; - if (PyDict_SetItemString(dict, n, self->ob_item[i]) < 0) + Py_ssize_t field = 0; + PyObject *key; + PyObject *value; + Py_ssize_t ix = 0; + while (PyDict_Next(Py_TYPE(self)->tp_dict, &ix, &key, &value)) { + if (Py_TYPE(value) != &PyMemberDescr_Type) { + continue; + } + PyMemberDescrObject *descr = (PyMemberDescrObject *)value; + if (field < n_visible_fields) { + ++field; + continue; + } + + PyObject* name = descr->d_common.d_name; + PyObject* memb = self->ob_item[field]; + if (PyDict_SetItem(dict, name, memb) < 0) goto error; + ++field; } result = Py_BuildValue("(O(OO))", Py_TYPE(self), tup, dict); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index e4952511e33a65..14210eb18bd9a4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1037,20 +1037,14 @@ PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) static int traverse_slots(PyTypeObject *type, PyObject *self, visitproc visit, void *arg) { - Py_ssize_t i, n; - PyMemberDef *mp; - - n = Py_SIZE(type); - mp = PyHeapType_GET_MEMBERS((PyHeapTypeObject *)type); - for (i = 0; i < n; i++, mp++) { - if (mp->type == T_OBJECT_EX) { - char *addr = (char *)self + mp->offset; - PyObject *obj = *(PyObject **)addr; - if (obj != NULL) { - int err = visit(obj, arg); - if (err) - return err; - } + _PyObject_MemberSlot *mp = _PyHeapType_GET_OBJECT_MEMBER_OFFSETS(type); + for (; mp->offset > 0; ++mp) { + char *addr = (char *)self + mp->offset; + PyObject *obj = *(PyObject **)addr; + if (obj != NULL) { + int err = visit(obj, arg); + if (err) + return err; } } return 0; @@ -1096,13 +1090,9 @@ subtype_traverse(PyObject *self, visitproc visit, void *arg) static void clear_slots(PyTypeObject *type, PyObject *self) { - Py_ssize_t i, n; - PyMemberDef *mp; - - n = Py_SIZE(type); - mp = PyHeapType_GET_MEMBERS((PyHeapTypeObject *)type); - for (i = 0; i < n; i++, mp++) { - if (mp->type == T_OBJECT_EX && !(mp->flags & READONLY)) { + _PyObject_MemberSlot *mp = _PyHeapType_GET_OBJECT_MEMBER_OFFSETS(type); + for (; mp->offset > 0; ++mp) { + if (!(mp->flags & READONLY)) { char *addr = (char *)self + mp->offset; PyObject *obj = *(PyObject **)addr; if (obj != NULL) { @@ -2326,7 +2316,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) PyObject *qualname, *slots = NULL, *tmp, *newslots, *cell; PyTypeObject *type = NULL, *base, *tmptype, *winner; PyHeapTypeObject *et; - PyMemberDef *mp; + PyMemberDef *tmp_members = NULL; Py_ssize_t i, nbases, nslots, slotoffset, name_size; int j, may_add_dict, may_add_weak, add_dict, add_weak; _Py_IDENTIFIER(__qualname__); @@ -2723,10 +2713,18 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) } /* Add descriptors for custom slots from __slots__, or for __dict__ */ - mp = PyHeapType_GET_MEMBERS(et); + _PyObject_MemberSlot *offsets = _PyHeapType_GET_OBJECT_MEMBER_OFFSETS(et); slotoffset = base->tp_basicsize; - if (et->ht_slots != NULL) { - for (i = 0; i < nslots; i++, mp++) { + if (et->ht_slots != NULL && nslots > 0) { + tmp_members = PyMem_RawCalloc(nslots + 1, sizeof(PyMemberDef)); + if (tmp_members == NULL) { + PyErr_NoMemory(); + goto error; + } + type->tp_members = tmp_members; + + for (i = 0; i < nslots; i++) { + PyMemberDef *mp = tmp_members + i; mp->name = PyUnicode_AsUTF8( PyTuple_GET_ITEM(et->ht_slots, i)); if (mp->name == NULL) @@ -2738,6 +2736,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) assert(strcmp(mp->name, "__dict__") != 0); assert(strcmp(mp->name, "__weakref__") != 0); + (offsets + i)->offset = slotoffset; slotoffset += sizeof(PyObject *); } } @@ -2755,7 +2754,6 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) } type->tp_basicsize = slotoffset; type->tp_itemsize = base->tp_itemsize; - type->tp_members = PyHeapType_GET_MEMBERS(et); if (type->tp_weaklistoffset && type->tp_dictoffset) type->tp_getset = subtype_getsets_full; @@ -2814,6 +2812,10 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) if (PyType_Ready(type) < 0) goto error; + type->tp_members = NULL; + PyMem_RawFree(tmp_members); + tmp_members = NULL; + /* Put the proper slots in place */ fixup_slot_dispatchers(type); @@ -2835,6 +2837,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) Py_XDECREF(bases); Py_XDECREF(slots); Py_XDECREF(type); + PyMem_RawFree(tmp_members); return NULL; } @@ -2860,7 +2863,9 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) if (slot->slot == Py_tp_members) { nmembers = 0; for (memb = slot->pfunc; memb->name != NULL; memb++) { - nmembers++; + if (memb->type == T_OBJECT || memb->type == T_OBJECT_EX) { + nmembers++; + } } } } @@ -2891,7 +2896,7 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) goto fail; res->ht_qualname = res->ht_name; Py_INCREF(res->ht_qualname); - type->tp_name = spec->name; + type->tp_name = PyUnicode_AsUTF8(res->ht_name); /* Adjust for empty tuple bases */ if (!bases) { @@ -2964,12 +2969,6 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) memcpy(tp_doc, old_doc, len); type->tp_doc = tp_doc; } - else if (slot->slot == Py_tp_members) { - /* Move the slots to the heap type itself */ - size_t len = Py_TYPE(type)->tp_itemsize * nmembers; - memcpy(PyHeapType_GET_MEMBERS(res), slot->pfunc, len); - type->tp_members = PyHeapType_GET_MEMBERS(res); - } else { /* Copy other slots directly */ *(void**)(res_start + slotoffsets[slot->slot]) = slot->pfunc; @@ -2985,6 +2984,13 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) if (PyType_Ready(type) < 0) goto fail; + + /* The memory behind these pointers might be reclaimed, don't leave + dangling pointers around. */ + type->tp_methods = NULL; + type->tp_getset = NULL; + type->tp_members = NULL; + if (type->tp_dictoffset) { res->ht_cached_keys = _PyDict_NewKeysForClass(); } @@ -3613,7 +3619,7 @@ PyTypeObject PyType_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "type", /* tp_name */ sizeof(PyHeapTypeObject), /* tp_basicsize */ - sizeof(PyMemberDef), /* tp_itemsize */ + sizeof(_PyObject_MemberSlot), /* tp_itemsize */ (destructor)type_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c3e30593475d9d..3e4a3237e2e9c3 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1812,8 +1812,8 @@ builtin_pow_impl(PyObject *module, PyObject *x, PyObject *y, PyObject *z) /* AC: cannot convert yet, waiting for *args support */ -static PyObject * -builtin_print(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +PyObject * +_PyBuiltin_Print(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { static const char * const _keywords[] = {"sep", "end", "file", "flush", 0}; static struct _PyArg_Parser _parser = {"|OOOO:print", _keywords, 0}; @@ -2740,7 +2740,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF BUILTIN_POW_METHODDEF - {"print", (PyCFunction)(void(*)(void))builtin_print, METH_FASTCALL | METH_KEYWORDS, print_doc}, + {"print", (PyCFunction)(void(*)(void))_PyBuiltin_Print, METH_FASTCALL | METH_KEYWORDS, print_doc}, BUILTIN_REPR_METHODDEF BUILTIN_ROUND_METHODDEF BUILTIN_SETATTR_METHODDEF diff --git a/Python/ceval.c b/Python/ceval.c index 60367a665d7d62..2c4137b88633f1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4880,7 +4880,7 @@ PyEval_GetFuncName(PyObject *func) else if (PyFunction_Check(func)) return PyUnicode_AsUTF8(((PyFunctionObject*)func)->func_name); else if (PyCFunction_Check(func)) - return ((PyCFunctionObject*)func)->m_ml->ml_name; + return PyUnicode_AsUTF8(((PyCFunctionObject*)func)->m_base.name); else return func->ob_type->tp_name; } diff --git a/Python/structmember.c b/Python/structmember.c index e653d0277c1a10..ae02d600beafc1 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -5,13 +5,13 @@ #include "structmember.h" -PyObject * -PyMember_GetOne(const char *addr, PyMemberDef *l) +static PyObject * +member_getone(const char *addr, Py_ssize_t offset, int type, const char *name) { PyObject *v; - addr += l->offset; - switch (l->type) { + addr += offset; + switch (type) { case T_BOOL: v = PyBool_FromLong(*(char*)addr); break; @@ -71,7 +71,7 @@ PyMember_GetOne(const char *addr, PyMemberDef *l) case T_OBJECT_EX: v = *(PyObject **)addr; if (v == NULL) - PyErr_SetString(PyExc_AttributeError, l->name); + PyErr_SetString(PyExc_AttributeError, name); Py_XINCREF(v); break; case T_LONGLONG: @@ -91,39 +91,60 @@ PyMember_GetOne(const char *addr, PyMemberDef *l) return v; } +PyObject * +PyMember_GetOne(const char *addr, struct PyMemberDef *memb) +{ + return member_getone(addr, memb->offset, memb->type, memb->name); +} + +PyObject * +_PyMemberDescr_GetOne(const char *addr, PyMemberDescrObject *memb) +{ + const char *name = PyUnicode_AsUTF8(memb->d_common.d_name); + if (name == NULL) { + return NULL; + } + return member_getone(addr, memb->d_offset, memb->d_member_type, name); +} + #define WARN(msg) \ do { \ if (PyErr_WarnEx(PyExc_RuntimeWarning, msg, 1) < 0) \ return -1; \ } while (0) -int -PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) +static int +member_setone(char *addr, + Py_ssize_t offset, + int flags, + int type, + const char *name, + PyObject *v) { PyObject *oldv; - addr += l->offset; + addr += offset; - if ((l->flags & READONLY)) + if ((flags & READONLY)) { PyErr_SetString(PyExc_AttributeError, "readonly attribute"); return -1; } if (v == NULL) { - if (l->type == T_OBJECT_EX) { + if (type == T_OBJECT_EX) { /* Check if the attribute is set. */ if (*(PyObject **)addr == NULL) { - PyErr_SetString(PyExc_AttributeError, l->name); + PyErr_SetString(PyExc_AttributeError, name); return -1; } } - else if (l->type != T_OBJECT) { + else if (type != T_OBJECT) { PyErr_SetString(PyExc_TypeError, "can't delete numeric/char attribute"); return -1; } } - switch (l->type) { + switch (type) { case T_BOOL:{ if (!PyBool_Check(v)) { PyErr_SetString(PyExc_TypeError, @@ -285,8 +306,29 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) } default: PyErr_Format(PyExc_SystemError, - "bad memberdescr type for %s", l->name); + "bad memberdescr type for %s", name); return -1; } return 0; } + +int +PyMember_SetOne(char *addr, struct PyMemberDef *memb, PyObject *v) +{ + return member_setone(addr, memb->offset, memb->flags, memb->type, memb->name, v); +} + +int +_PyMemberDescr_SetOne(char *addr, PyMemberDescrObject *memb, PyObject *v) +{ + const char *name = PyUnicode_AsUTF8(memb->d_common.d_name); + if (name == NULL) { + return -1; + } + return member_setone(addr, + memb->d_offset, + memb->d_flags, + memb->d_member_type, + name, + v); +} diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 93f720ab7e2a10..801af8fa02cfcf 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -593,20 +593,20 @@ class PyClassObjectPtr(PyObjectPtr): class BuiltInFunctionProxy(object): - def __init__(self, ml_name): - self.ml_name = ml_name + def __init__(self, name): + self.name = name def __repr__(self): - return "" % self.ml_name + return "" % self.name class BuiltInMethodProxy(object): - def __init__(self, ml_name, pyop_m_self): - self.ml_name = ml_name + def __init__(self, name, pyop_m_self): + self.name = name self.pyop_m_self = pyop_m_self def __repr__(self): return ('' - % (self.ml_name, + % (self.name, self.pyop_m_self.safe_tp_name(), self.pyop_m_self.as_address()) ) @@ -619,17 +619,15 @@ class PyCFunctionObjectPtr(PyObjectPtr): _typename = 'PyCFunctionObject' def proxyval(self, visited): - m_ml = self.field('m_ml') # m_ml is a (PyMethodDef*) - try: - ml_name = m_ml['ml_name'].string() - except UnicodeDecodeError: - ml_name = '' + m_base = self.field('m_base') + name_ob = PyUnicodeObjectPtr.from_pyobject_ptr(m_base['name']) + name = name_ob.proxyval(set()) pyop_m_self = self.pyop_field('m_self') if pyop_m_self.is_null(): - return BuiltInFunctionProxy(ml_name) + return BuiltInFunctionProxy(name) else: - return BuiltInMethodProxy(ml_name, pyop_m_self) + return BuiltInMethodProxy(name, pyop_m_self) class PyCodeObjectPtr(PyObjectPtr):