diff --git a/.azure/build.yml b/.azure/build.yml index 1ebe24559..80b564d1b 100644 --- a/.azure/build.yml +++ b/.azure/build.yml @@ -71,6 +71,10 @@ jobs: imageName: "ubuntu-latest" python.version: '3.12' jdk.version: '17' + linux-py3.13-jdk17: + imageName: "ubuntu-latest" + python.version: "3.13.0-rc.2" + jdk.version: '17' # Windows windows-py3.8-jdk8: imageName: "windows-2019" diff --git a/native/python/pyjp_class.cpp b/native/python/pyjp_class.cpp index 72a1ead45..5aaa9a24b 100644 --- a/native/python/pyjp_class.cpp +++ b/native/python/pyjp_class.cpp @@ -58,82 +58,18 @@ static int PyJPClass_clear(PyJPClass *self) return 0; } -PyObject *PyJPClass_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - JP_PY_TRY("PyJPClass_new"); - if (PyTuple_Size(args) != 3) - JP_RAISE(PyExc_TypeError, "Java class meta required 3 arguments"); - - JP_BLOCK("PyJPClass_new::verify") - { - // Watch for final classes - PyObject *bases = PyTuple_GetItem(args, 1); - Py_ssize_t len = PyTuple_Size(bases); - for (Py_ssize_t i = 0; i < len; ++i) - { - PyObject *item = PyTuple_GetItem(bases, i); - JPClass *cls = PyJPClass_getJPClass(item); - if (cls != nullptr && cls->isFinal()) - { - PyErr_Format(PyExc_TypeError, "Cannot extend final class '%s'", - ((PyTypeObject*) item)->tp_name); - } - } - } - - int magic = 0; - if (kwargs == PyJPClassMagic || (kwargs != nullptr && PyDict_GetItemString(kwargs, "internal") != nullptr)) - { - magic = 1; - kwargs = nullptr; - } - if (magic == 0) - { - PyErr_Format(PyExc_TypeError, "Java classes cannot be extended in Python"); - return nullptr; - } - - auto *typenew = (PyTypeObject*) PyType_Type.tp_new(type, args, kwargs); - - // GCOVR_EXCL_START - // Sanity checks. Not testable - if (typenew == nullptr) - return nullptr; - if (typenew->tp_finalize != nullptr && typenew->tp_finalize != (destructor) PyJPValue_finalize) - { - Py_DECREF(typenew); - PyErr_SetString(PyExc_TypeError, "finalizer conflict"); - return nullptr; - } - - // This sanity check is trigger if the user attempts to build their own - // type wrapper with a __del__ method defined. It is hard to trigger. - if (typenew->tp_alloc != (allocfunc) PyJPValue_alloc - && typenew->tp_alloc != PyBaseObject_Type.tp_alloc) - { - Py_DECREF(typenew); - PyErr_SetString(PyExc_TypeError, "alloc conflict"); - return nullptr; - } - // GCOVR_EXCL_STOP - - typenew->tp_alloc = (allocfunc) PyJPValue_alloc; - typenew->tp_finalize = (destructor) PyJPValue_finalize; - - if (PyObject_IsSubclass((PyObject*) typenew, (PyObject*) PyJPException_Type)) - { - typenew->tp_new = PyJPException_Type->tp_new; - } - ((PyJPClass*) typenew)->m_Doc = nullptr; - return (PyObject*) typenew; - JP_PY_CATCH(nullptr); -} - PyObject* examine(PyObject *module, PyObject *other); PyObject* PyJPClass_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { JP_PY_TRY("PyJPClass_FromSpecWithBases"); +#if PY_VERSION_HEX>=0x030c0000 + // Starting in Python 3.12 there is a function for creating from a meta class + // that replaces this madeness. + PyTypeObject *type = (PyTypeObject*) PyType_FromMetaclass((PyTypeObject*) PyJPClass_Type, NULL, spec, bases); + if (type == nullptr) + return (PyObject*) type; +#else // Python lacks a FromSpecWithMeta so we are going to have to fake it here. auto* type = (PyTypeObject*) PyJPClass_Type->tp_alloc(PyJPClass_Type, 0); auto* heap = (PyHeapTypeObject*) type; @@ -177,6 +113,12 @@ PyObject* PyJPClass_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { switch (slot->slot) { + case Py_tp_finalize: + type->tp_finalize = (destructor) slot->pfunc; + break; + case Py_tp_alloc: + type->tp_alloc = (allocfunc) slot->pfunc; + break; case Py_tp_free: type->tp_free = (freefunc) slot->pfunc; break; @@ -296,7 +238,7 @@ PyObject* PyJPClass_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) } // GC objects are required to implement clear and traverse, this is a - // safety check to make sure we implemented all properly. This error should + // safety check to make sure we implemented all properly. This error should // never happen in production code. if (PyType_IS_GC(type) && ( type->tp_traverse==nullptr || @@ -305,6 +247,13 @@ PyObject* PyJPClass_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) PyErr_Format(PyExc_TypeError, "GC requirements failed for %s", spec->name); JP_RAISE_PYTHON(); } + +#endif + + // Make sure our memory model is used + type->tp_alloc = (allocfunc) PyJPValue_alloc; + type->tp_finalize = (destructor) PyJPValue_finalize; + PyType_Ready(type); PyDict_SetItemString(type->tp_dict, "__module__", PyUnicode_FromString("_jpype")); return (PyObject*) type; @@ -314,8 +263,34 @@ PyObject* PyJPClass_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) int PyJPClass_init(PyObject *self, PyObject *args, PyObject *kwargs) { JP_PY_TRY("PyJPClass_init"); - if (PyTuple_Size(args) == 1) - return 0; + + if (!PyObject_IsInstance(self, (PyObject*) PyJPClass_Type)) + { + PyErr_SetString(PyExc_TypeError, "Type incorrect"); + return -1; + } + + PyTypeObject *type = (PyTypeObject*) self; + +#if PY_VERSION_HEX >= 0x030d0000 + // Python 3.13 - This flag will try to place the dictionary are part of the object which + // adds an unknown number of bytes to the end of the object making it impossible + // to attach our needed data. If we kill the flag then we get usable behavior. + type->tp_flags &= ~Py_TPFLAGS_INLINE_VALUES; +#endif + + // Verify that we were called internally + int magic = 0; + if (kwargs == PyJPClassMagic || (kwargs != nullptr && PyDict_GetItemString(kwargs, "internal") != nullptr)) + { + magic = 1; + kwargs = nullptr; + } + if (magic == 0) + { + PyErr_Format(PyExc_TypeError, "Java classes cannot be extended in Python"); + return -1; + } // Set the host object PyObject *name = nullptr; @@ -330,20 +305,61 @@ int PyJPClass_init(PyObject *self, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_TypeError, "Bases must be a tuple"); return -1; } - for (int i = 0; i < PyTuple_Size(bases); ++i) + + JP_BLOCK("PyJPClass_new::verify") { - if (!PyJPClass_Check(PyTuple_GetItem(bases, i))) + // Watch for final classes + Py_ssize_t len = PyTuple_Size(bases); + for (Py_ssize_t i = 0; i < len; ++i) { - PyErr_SetString(PyExc_TypeError, "All bases must be Java types"); - return -1; + PyObject *item = PyTuple_GetItem(bases, i); + JPClass *cls = PyJPClass_getJPClass(item); + if (cls != nullptr && cls->isFinal()) + { + PyErr_Format(PyExc_TypeError, "Cannot extend final class '%s'", + ((PyTypeObject*) item)->tp_name); + } } } + // We must make sure that all classes have our allocator + type->tp_alloc = (allocfunc) PyJPValue_alloc; + type->tp_finalize = (destructor) PyJPValue_finalize; + ((PyJPClass*) self)->m_Doc = nullptr; + // Call the type init int rc = PyType_Type.tp_init(self, args, nullptr); if (rc == -1) return rc; // GCOVR_EXCL_LINE no clue how to trigger this one + // GCOVR_EXCL_START + // Sanity checks. Not testable + if (type == nullptr) + return -1; + if (type->tp_finalize != nullptr && type->tp_finalize != (destructor) PyJPValue_finalize) + { + PyErr_SetString(PyExc_TypeError, "finalizer conflict"); + return -1; + } + + // This sanity check is trigger if the user attempts to build their own + // type wrapper with a __del__ method defined. It is hard to trigger. + if (type->tp_alloc != (allocfunc) PyJPValue_alloc + && type->tp_alloc != PyBaseObject_Type.tp_alloc) + { + PyErr_SetString(PyExc_TypeError, "alloc conflict"); + return -1; + } + // GCOVR_EXCL_STOP + +#if PY_VERSION_HEX < 0x03090000 + // This was required at one point but I don't know what version it applied to. + if (PyObject_IsSubclass((PyObject*) type, (PyObject*) PyJPException_Type)) + { + type->tp_new = PyJPException_Type->tp_new; + } +#endif + return rc; JP_PY_CATCH(-1); } @@ -993,7 +1009,6 @@ static PyGetSetDef classGetSets[] = { static PyType_Slot classSlots[] = { { Py_tp_alloc, (void*) PyJPValue_alloc}, { Py_tp_finalize, (void*) PyJPValue_finalize}, - { Py_tp_new, (void*) PyJPClass_new}, { Py_tp_init, (void*) PyJPClass_init}, { Py_tp_dealloc, (void*) PyJPClass_dealloc}, { Py_tp_traverse, (void*) PyJPClass_traverse}, @@ -1132,7 +1147,7 @@ JPPyObject PyJPClass_getBases(JPJavaFrame &frame, JPClass* cls) * Internal method for wrapping a returned Java class instance. * * This checks the cache for existing wrappers and then - * transfers control to JClassFactory. This is required because all of + * transfers control to JClassFactory. This is required because all of * the post load stuff needs to be in Python. * * @param cls @@ -1203,7 +1218,7 @@ void PyJPClass_hook(JPJavaFrame &frame, JPClass* cls) JP_TRACE("type new"); // Create the type using the meta class magic - JPPyObject vself = JPPyObject::call(PyJPClass_Type->tp_new(PyJPClass_Type, rc.get(), PyJPClassMagic)); + JPPyObject vself = JPPyObject::call(PyJPClass_Type->tp_call((PyObject*) PyJPClass_Type, rc.get(), PyJPClassMagic)); auto *self = (PyJPClass*) vself.get(); // Attach the javaSlot diff --git a/native/python/pyjp_module.cpp b/native/python/pyjp_module.cpp index a2110efea..80d87f611 100644 --- a/native/python/pyjp_module.cpp +++ b/native/python/pyjp_module.cpp @@ -37,6 +37,7 @@ extern void PyJPNumber_initType(PyObject* module); extern void PyJPClassHints_initType(PyObject* module); extern void PyJPPackage_initType(PyObject* module); extern void PyJPChar_initType(PyObject* module); +extern void PyJPValue_initType(PyObject* module); static PyObject *PyJPModule_convertBuffer(JPPyBuffer& buffer, PyObject *dtype); @@ -739,6 +740,7 @@ PyMODINIT_FUNC PyInit__jpype() PyJPClassMagic = PyDict_New(); // Initialize each of the python extension types + PyJPValue_initType(module); PyJPClass_initType(module); PyJPObject_initType(module); diff --git a/native/python/pyjp_number.cpp b/native/python/pyjp_number.cpp index 9f55af4a1..dfede7f5b 100644 --- a/native/python/pyjp_number.cpp +++ b/native/python/pyjp_number.cpp @@ -298,6 +298,17 @@ static PyObject* PyJPBoolean_str(PyObject* self) JP_PY_CATCH(nullptr); } +static PyObject *PyJPNumber_initSubclass(PyObject *cls, PyObject* args, PyObject *kwargs) +{ + Py_RETURN_NONE; +} + +static PyMethodDef numberMethods[] = { + {"__init_subclass__", (PyCFunction) PyJPNumber_initSubclass, METH_CLASS | METH_VARARGS | METH_KEYWORDS, ""}, + {0} +}; + + static PyType_Slot numberLongSlots[] = { {Py_tp_new, (void*) &PyJPNumber_new}, {Py_tp_getattro, (void*) &PyJPValue_getattro}, @@ -308,6 +319,7 @@ static PyType_Slot numberLongSlots[] = { {Py_tp_repr, (void*) &PyJPNumberLong_repr}, {Py_tp_hash, (void*) &PyJPNumberLong_hash}, {Py_tp_richcompare, (void*) &PyJPNumberLong_compare}, + {Py_tp_methods, (void*) numberMethods}, {0} }; @@ -330,6 +342,7 @@ static PyType_Slot numberFloatSlots[] = { {Py_tp_repr, (void*) &PyJPNumberFloat_repr}, {Py_tp_hash, (void*) &PyJPNumberFloat_hash}, {Py_tp_richcompare, (void*) &PyJPNumberFloat_compare}, + {Py_tp_methods, (void*) numberMethods}, {0} }; @@ -352,6 +365,7 @@ static PyType_Slot numberBooleanSlots[] = { {Py_nb_float, (void*) PyJPNumberLong_float}, {Py_tp_hash, (void*) PyJPNumberLong_hash}, {Py_tp_richcompare, (void*) PyJPNumberLong_compare}, + {Py_tp_methods, (void*) numberMethods}, {0} }; diff --git a/native/python/pyjp_object.cpp b/native/python/pyjp_object.cpp index 46fe6d892..0414a4151 100644 --- a/native/python/pyjp_object.cpp +++ b/native/python/pyjp_object.cpp @@ -15,7 +15,6 @@ *****************************************************************************/ #include "jpype.h" #include "pyjp.h" -#include #ifdef __cplusplus extern "C" @@ -222,7 +221,19 @@ static PyObject *PyJPObject_repr(PyObject *self) JP_PY_CATCH(nullptr); // GCOVR_EXCL_LINE } +static PyObject *PyJPObject_initSubclass(PyObject *cls, PyObject* args, PyObject *kwargs) +{ + Py_RETURN_NONE; +} + +static PyMethodDef objectMethods[] = { + {"__init_subclass__", (PyCFunction) PyJPObject_initSubclass, METH_CLASS | METH_VARARGS | METH_KEYWORDS, ""}, + {0} +}; + static PyType_Slot objectSlots[] = { + {Py_tp_alloc, (void*) &PyJPValue_alloc}, + {Py_tp_finalize, (void*) &PyJPValue_finalize}, {Py_tp_new, (void*) &PyJPObject_new}, {Py_tp_free, (void*) &PyJPValue_free}, {Py_tp_getattro, (void*) &PyJPValue_getattro}, @@ -231,6 +242,7 @@ static PyType_Slot objectSlots[] = { {Py_tp_repr, (void*) &PyJPObject_repr}, {Py_tp_richcompare, (void*) &PyJPObject_compare}, {Py_tp_hash, (void*) &PyJPObject_hash}, + {Py_tp_methods, (void*) objectMethods}, {0} }; @@ -361,12 +373,11 @@ static PyType_Spec comparableSpec = { void PyJPObject_initType(PyObject* module) { - PyJPObject_Type = (PyTypeObject*) PyJPClass_FromSpecWithBases(&objectSpec, nullptr); - JP_PY_CHECK(); // GCOVR_EXCL_LINE + PyJPObject_Type = (PyTypeObject*) PyJPClass_FromSpecWithBases(&objectSpec, nullptr); + JP_PY_CHECK(); // GCOVR_EXCL_LINE PyModule_AddObject(module, "_JObject", (PyObject*) PyJPObject_Type); JP_PY_CHECK(); // GCOVR_EXCL_LINE - - JPPyObject bases = JPPyTuple_Pack(PyExc_Exception, PyJPObject_Type); + JPPyObject bases = JPPyTuple_Pack(PyExc_Exception, PyJPObject_Type); PyJPException_Type = (PyTypeObject*) PyJPClass_FromSpecWithBases(&excSpec, bases.get()); JP_PY_CHECK(); // GCOVR_EXCL_LINE PyModule_AddObject(module, "_JException", (PyObject*) PyJPException_Type); diff --git a/native/python/pyjp_value.cpp b/native/python/pyjp_value.cpp index 1deecaa92..f3b167d6c 100644 --- a/native/python/pyjp_value.cpp +++ b/native/python/pyjp_value.cpp @@ -16,12 +16,19 @@ #include "jpype.h" #include "pyjp.h" #include "jp_stringtype.h" +#include +#include #ifdef __cplusplus extern "C" { #endif +std::mutex mtx; + +// Create a dummy type which we will use only for allocation +PyTypeObject* PyJPAlloc_Type = nullptr; + /** * Allocate a new Python object with a slot for Java. * @@ -40,53 +47,37 @@ extern "C" PyObject* PyJPValue_alloc(PyTypeObject* type, Py_ssize_t nitems) { JP_PY_TRY("PyJPValue_alloc"); - // Modification from Python to add size elements - const size_t size = _PyObject_VAR_SIZE(type, nitems + 1) + sizeof (JPValue); - PyObject *obj = nullptr; - if (PyType_IS_GC(type)) - { - // Horrible kludge because python lacks an API for allocating a GC type with extra memory - // The private method _PyObject_GC_Alloc is no longer visible, so we are forced to allocate - // a different type with the extra memory and then hot swap the type to the real one. - PyTypeObject type2; - type2.tp_basicsize = size; - type2.tp_itemsize = 0; - type2.tp_name = nullptr; - type2.tp_flags = type->tp_flags; - type2.tp_traverse = type->tp_traverse; - - // Allocate the fake type - obj = PyObject_GC_New(PyObject, &type2); - - // Note the object will be inited twice which should not leak. (fingers crossed) + +#if PY_VERSION_HEX >= 0x030d0000 + // This flag will try to place the dictionary are part of the object which + // adds an unknown number of bytes to the end of the object making it impossible + // to attach our needed data. If we kill the flag then we get usable behavior. + if (PyType_HasFeature(type, Py_TPFLAGS_INLINE_VALUES)) { + PyErr_Format(PyExc_RuntimeError, "Unhandled object layout"); + return 0; } - else - { - obj = (PyObject*) PyObject_MALLOC(size); +#endif + + PyObject* obj = nullptr; + { + std::lock_guard lock(mtx); + // Mutate the allocator type + PyJPAlloc_Type->tp_flags = type->tp_flags; + PyJPAlloc_Type->tp_basicsize = type->tp_basicsize + sizeof (JPValue); + PyJPAlloc_Type->tp_itemsize = type->tp_itemsize; + + // Create a new allocation for the dummy type + obj = PyType_GenericAlloc(PyJPAlloc_Type, nitems); } - if (obj == nullptr) - return PyErr_NoMemory(); // GCOVR_EXCL_LINE - memset(obj, 0, size); + // Watch for memory errors + if (obj == nullptr) + return nullptr; - Py_ssize_t refcnt = ((PyObject*) type)->ob_refcnt; + // Polymorph the type to match our desired type obj->ob_type = type; + Py_INCREF(type); - if (type->tp_itemsize == 0) - PyObject_Init(obj, type); - else - PyObject_InitVar((PyVarObject *) obj, type, nitems); - - // This line is required to deal with Python bug (GH-11661) - // Some versions of Python fail to increment the reference counter of - // heap types properly. - if (refcnt == ((PyObject*) type)->ob_refcnt) - Py_INCREF(type); // GCOVR_EXCL_LINE - - if (PyType_IS_GC(type)) - { - PyObject_GC_Track(obj); - } JP_TRACE("alloc", type->tp_name, obj); return obj; JP_PY_CATCH(nullptr); @@ -107,17 +98,20 @@ Py_ssize_t PyJPValue_getJavaSlotOffset(PyObject* self) if (type == nullptr || type->tp_alloc != (allocfunc) PyJPValue_alloc || type->tp_finalize != (destructor) PyJPValue_finalize) + { return 0; - Py_ssize_t offset; - Py_ssize_t sz = 0; + } + Py_ssize_t offset = 0; + Py_ssize_t sz = 0; + #if PY_VERSION_HEX>=0x030c0000 // starting in 3.12 there is no longer ob_size in PyLong if (PyType_HasFeature(self->ob_type, Py_TPFLAGS_LONG_SUBCLASS)) sz = (((PyLongObject*)self)->long_value.lv_tag) >> 3; // Private NON_SIZE_BITS else #endif - if (type->tp_itemsize != 0) + if (type->tp_itemsize != 0) sz = Py_SIZE(self); // PyLong abuses ob_size with negative values prior to 3.12 if (sz < 0) @@ -221,7 +215,7 @@ PyObject* PyJPValue_str(PyObject* self) Py_INCREF(cache); return cache; } - auto jstr = (jstring) value->getValue().l; + auto jstr = (jstring) value->getValue().l; string str; str = frame.toStringUTF8(jstr); cache = JPPyString::fromStringUTF8(str).keep(); @@ -337,3 +331,38 @@ bool PyJPValue_isSetJavaSlot(PyObject* self) auto* slot = (JPValue*) (((char*) self) + offset); return slot->getClass() != nullptr; } + +/***************** Create a dummy type for use when allocating. ************************/ +static int PyJPAlloc_traverse(PyObject *self, visitproc visit, void *arg) +{ + return 0; +} + +static int PyJPAlloc_clear(PyObject *self) +{ + return 0; +} + + +static PyType_Slot allocSlots[] = { + { Py_tp_traverse, (void*) PyJPAlloc_traverse}, + { Py_tp_clear, (void*) PyJPAlloc_clear}, + {0, NULL} // Sentinel +}; + +static PyType_Spec allocSpec = { + "_jpype._JAlloc", + sizeof(PyObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + allocSlots +}; + +void PyJPValue_initType(PyObject* module) +{ + PyObject *bases = PyTuple_Pack(1, &PyBaseObject_Type); + PyJPAlloc_Type = (PyTypeObject*) PyType_FromSpecWithBases(&allocSpec, bases); + Py_DECREF(bases); + Py_INCREF(PyJPAlloc_Type); + JP_PY_CHECK(); +} diff --git a/test/jpypetest/test_fault.py b/test/jpypetest/test_fault.py index 441aa5466..c84ab3e30 100644 --- a/test/jpypetest/test_fault.py +++ b/test/jpypetest/test_fault.py @@ -92,16 +92,6 @@ def testJPObject_null(self): null.int_field = 1 # null.callInt(1) - @common.requireInstrumentation - def testJPClass_new(self): - _jpype.fault("PyJPClass_new") - with self.assertRaisesRegex(SystemError, "fault"): - _jpype._JClass("foo", (object,), {}) - with self.assertRaises(TypeError): - _jpype._JClass("foo", (object,), {}) - with self.assertRaises(TypeError): - _jpype._JClass("foo", (_jpype._JObject,), {'__del__': None}) - @common.requireInstrumentation def testJPClass_init(self): _jpype.fault("PyJPClass_init")