From 21ef638651f97b45e62e58bba08ef5b386632ef1 Mon Sep 17 00:00:00 2001 From: "Karl E. Nelson" Date: Thu, 1 Sep 2022 13:27:11 -0700 Subject: [PATCH 1/8] Fixes for 3.11 --- native/common/jp_exception.cpp | 46 ++++++++++++++-------------------- native/python/pyjp_value.cpp | 4 ++- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/native/common/jp_exception.cpp b/native/common/jp_exception.cpp index 477e99a86..53b7185ba 100644 --- a/native/common/jp_exception.cpp +++ b/native/common/jp_exception.cpp @@ -489,18 +489,18 @@ void JPypeException::toJava(JPContext *context) JP_TRACE_OUT; // GCOVR_EXCL_LINE } -PyTracebackObject *tb_create( - PyTracebackObject *last_traceback, +PyObject *tb_create( + PyObject *last_traceback, PyObject *dict, const char* filename, const char* funcname, int linenum) { // Create a code for this frame. (ref count is 1) - PyCodeObject *code = PyCode_NewEmpty(filename, funcname, linenum); + JPPyObject code = JPPyObject::accept((PyObject*)PyCode_NewEmpty(filename, funcname, linenum)); // If we don't get the code object there is no point - if (code == NULL) + if (code.get() == NULL) return NULL; // This is a bit of a kludge. Python lacks a way to directly create @@ -520,48 +520,40 @@ PyTracebackObject *tb_create( // portable, but we have to create a big (uninitialized object) each time we // want to pass in the previous frame. PyThreadState state; + JPPyObject prev; if (last_traceback != NULL) - state.frame = last_traceback->tb_frame; + { + prev = JPPyObject::call(PyObject_GetAttrString(last_traceback, "tb_frame")); + state.frame = (PyFrameObject*) prev.get(); + } else state.frame = NULL; // Create a frame for the traceback. - PyFrameObject *frame = PyFrame_New(&state, code, dict, NULL); - - // frame just borrows the reference rather than claiming it - // so we need to get rid of the extra reference here. - Py_DECREF(code); + JPPyObject frame = JPPyObject::call((PyObject*) PyFrame_New(&state, (PyCodeObject*) code.get(), dict, NULL)); // If we don't get the frame object there is no point - if (frame == NULL) + if (frame.get() == NULL) return NULL; // Create a traceback - PyTracebackObject *traceback = (PyTracebackObject*) - PyObject_GC_New(PyTracebackObject, &PyTraceBack_Type); + JPPyObject lasti = JPPyObject::claim(PyLong_FromLong(((PyFrameObject*)frame.get())->f_lasti)); + JPPyObject linenuma = JPPyObject::claim(PyLong_FromLong(linenum)); + JPPyObject tuple = JPPyObject::call(PyTuple_Pack(4, Py_None, frame.get(), lasti.get(), linenuma.get())); + JPPyObject traceback = JPPyObject::accept(PyObject_Call((PyObject*) &PyTraceBack_Type, tuple.get(), NULL)); // We could fail in process - if (traceback == NULL) + if (traceback.get() == NULL) { - Py_DECREF(frame); return NULL; } - // Set the fields - traceback->tb_frame = frame; // Steal the reference from frame - traceback->tb_lasti = frame->f_lasti; - traceback->tb_lineno = linenum; - Py_XINCREF(last_traceback); - traceback->tb_next = last_traceback; - - // Allow GC on the object - PyObject_GC_Track(traceback); - return traceback; + return traceback.keep(); } PyObject* PyTrace_FromJPStackTrace(JPStackTrace& trace) { - PyTracebackObject *last_traceback = NULL; + PyObject *last_traceback = NULL; PyObject *dict = PyModule_GetDict(PyJPModule); for (JPStackTrace::iterator iter = trace.begin(); iter != trace.end(); ++iter) { @@ -575,7 +567,7 @@ PyObject* PyTrace_FromJPStackTrace(JPStackTrace& trace) JPPyObject PyTrace_FromJavaException(JPJavaFrame& frame, jthrowable th, jthrowable prev) { - PyTracebackObject *last_traceback = NULL; + PyObject *last_traceback = NULL; JPContext *context = frame.getContext(); jvalue args[2]; args[0].l = th; diff --git a/native/python/pyjp_value.cpp b/native/python/pyjp_value.cpp index 6947a7631..366e00193 100644 --- a/native/python/pyjp_value.cpp +++ b/native/python/pyjp_value.cpp @@ -23,6 +23,8 @@ extern "C" { #endif +extern PyObject * _PyObject_GC_Malloc(size_t basicsize); + /** * Allocate a new Python object with a slot for Java. * @@ -305,4 +307,4 @@ bool PyJPValue_isSetJavaSlot(PyObject* self) return false; // GCOVR_EXCL_LINE JPValue* slot = (JPValue*) (((char*) self) + offset); return slot->getClass() != NULL; -} \ No newline at end of file +} From 05b02ffd9a8b6855904cc70faaf3d5359c888e5b Mon Sep 17 00:00:00 2001 From: "Karl E. Nelson" Date: Fri, 7 Oct 2022 07:09:14 -0700 Subject: [PATCH 2/8] Work on Python 3.11 --- native/common/jp_context.cpp | 6 +----- native/common/jp_exception.cpp | 32 +++++++++----------------------- native/python/pyjp_value.cpp | 32 +++++++++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/native/common/jp_context.cpp b/native/common/jp_context.cpp index 40a017216..7e1184a06 100644 --- a/native/common/jp_context.cpp +++ b/native/common/jp_context.cpp @@ -282,14 +282,10 @@ void JPContext::initializeResources(JNIEnv* env, bool interrupt) if (!m_Embedded) { - JPPyObject import = JPPyObject::call(PyImport_AddModule("importlib.util")); + JPPyObject import = JPPyObject::use(PyImport_AddModule("importlib.util")); JPPyObject jpype = JPPyObject::call(PyObject_CallMethod(import.get(), "find_spec", "s", "_jpype")); JPPyObject origin = JPPyObject::call(PyObject_GetAttrString(jpype.get(), "origin")); val[2].l = frame.fromStringUTF8(JPPyString::asStringUTF8(origin.get())); - import.incref(); // The documentation specifies that PyImport_AddModule must return a - // new reference, but that is not happening in Python 3.10 - // so we are triggering a gc assertion failure. To prevent - // the failure manually up the reference counter here. } m_JavaContext = JPObjectRef(frame, frame.CallStaticObjectMethodA(contextClass, startMethod, val)); diff --git a/native/common/jp_exception.cpp b/native/common/jp_exception.cpp index 477e99a86..f0e0fc71e 100644 --- a/native/common/jp_exception.cpp +++ b/native/common/jp_exception.cpp @@ -503,30 +503,16 @@ PyTracebackObject *tb_create( if (code == NULL) return NULL; - // This is a bit of a kludge. Python lacks a way to directly create - // a frame from a code object except when creating from the threadstate. - // - // In reviewing Python implementation, I find that the only element accessed - // in the thread state was the previous frame to link to. Because frame - // objects change a lot between different Python versions, trying to - // replicate the actions of setting up a frame is difficult to keep portable. - // - // Python 3.10 introduces the additional requirement that the global - // dictionary supplied must have a __builtins__. We can do this once - // when create the module. - // - // If instead we create a thread state and point the field it needs to the - // previous frame we create the frames using the defined API. Much more - // portable, but we have to create a big (uninitialized object) each time we - // want to pass in the previous frame. - PyThreadState state; - if (last_traceback != NULL) - state.frame = last_traceback->tb_frame; - else - state.frame = NULL; // Create a frame for the traceback. - PyFrameObject *frame = PyFrame_New(&state, code, dict, NULL); + PyThreadState *state = PyThreadState_GET(); + PyFrameObject *frame = PyFrame_New(state, code, dict, NULL); + + // Swap the back pointer + //PyObject* old = frame->f_back; + //frame->f_back = last_traceback->tb_frame; + //Py_XDECREF(old); + //Py_XINCREF(last_traceback->tb_frame); // frame just borrows the reference rather than claiming it // so we need to get rid of the extra reference here. @@ -549,7 +535,7 @@ PyTracebackObject *tb_create( // Set the fields traceback->tb_frame = frame; // Steal the reference from frame - traceback->tb_lasti = frame->f_lasti; + traceback->tb_lasti = PyFrame_GetLasti(frame); traceback->tb_lineno = linenum; Py_XINCREF(last_traceback); traceback->tb_next = last_traceback; diff --git a/native/python/pyjp_value.cpp b/native/python/pyjp_value.cpp index 6947a7631..86398a312 100644 --- a/native/python/pyjp_value.cpp +++ b/native/python/pyjp_value.cpp @@ -43,8 +43,34 @@ PyObject* PyJPValue_alloc(PyTypeObject* type, Py_ssize_t nitems ) JP_PY_TRY("PyJPValue_alloc"); // Modification from Python to add size elements const size_t size = _PyObject_VAR_SIZE(type, nitems + 1) + sizeof (JPValue); - PyObject *obj = (PyType_IS_GC(type)) ? _PyObject_GC_Malloc(size) - : (PyObject *) PyObject_MALLOC(size); + PyObject *obj = NULL; + if (PyType_IS_GC(type)) + { + // Horrible kludge because python lacks an API for allocating a GC type with extra memory + // The privor method _PyObject_GC_Alloc is no longer visible, so we are forced to allocate + // a different type with the extra memory and then hot swap the type to the real one. + PyTypeObject type2; + type2.tp_basicsize = size; + type2.tp_itemsize = 0; + type2.tp_name = NULL; + type2.tp_flags = type->tp_flags; + type2.tp_traverse = type->tp_traverse; + + // Allocate the fake type + obj = PyObject_GC_New(PyObject, &type2); + + // Hot swap to the required type + obj->ob_type = type; + if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) + { + Py_INCREF(type); + } + // Note the object will be inited twice which should not leak. (fingers crossed) + } + else + { + obj = (PyObject*) PyObject_MALLOC(size); + } if (obj == NULL) return PyErr_NoMemory(); // GCOVR_EXCL_LINE memset(obj, 0, size); @@ -305,4 +331,4 @@ bool PyJPValue_isSetJavaSlot(PyObject* self) return false; // GCOVR_EXCL_LINE JPValue* slot = (JPValue*) (((char*) self) + offset); return slot->getClass() != NULL; -} \ No newline at end of file +} From 68c246f0fd9f03599af08fa70b6655de6c4b409c Mon Sep 17 00:00:00 2001 From: "Karl E. Nelson" Date: Fri, 7 Oct 2022 13:11:19 -0700 Subject: [PATCH 3/8] Remove extra reference --- native/python/pyjp_value.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/native/python/pyjp_value.cpp b/native/python/pyjp_value.cpp index 65327a0d5..e9ac16637 100644 --- a/native/python/pyjp_value.cpp +++ b/native/python/pyjp_value.cpp @@ -61,12 +61,6 @@ PyObject* PyJPValue_alloc(PyTypeObject* type, Py_ssize_t nitems ) // Allocate the fake type obj = PyObject_GC_New(PyObject, &type2); - // Hot swap to the required type - obj->ob_type = type; - if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) - { - Py_INCREF(type); - } // Note the object will be inited twice which should not leak. (fingers crossed) } else From 42716c1f88ff359f282c4598c1962b9598d8a3b9 Mon Sep 17 00:00:00 2001 From: "Karl E. Nelson" Date: Mon, 10 Oct 2022 09:39:25 -0700 Subject: [PATCH 4/8] Minor fixes --- jpype/_jvmfinder.py | 6 +++--- setupext/test_java.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jpype/_jvmfinder.py b/jpype/_jvmfinder.py index e738ee48b..64b656c2b 100644 --- a/jpype/_jvmfinder.py +++ b/jpype/_jvmfinder.py @@ -320,10 +320,10 @@ def _javahome_binary(self): """ import platform import subprocess - from distutils.version import StrictVersion + from packaging.version import Version - current = StrictVersion(platform.mac_ver()[0][:4]) - if current >= StrictVersion('10.6') and current < StrictVersion('10.9'): + current = Version(platform.mac_ver()[0][:4]) + if current >= Version('10.6') and current < Version('10.9'): return subprocess.check_output( ['/usr/libexec/java_home']).strip() diff --git a/setupext/test_java.py b/setupext/test_java.py index d0a02dec5..5e8383a36 100644 --- a/setupext/test_java.py +++ b/setupext/test_java.py @@ -53,7 +53,7 @@ def compileJava(): srcs = glob.glob('test/harness/jpype/**/*.java', recursive=True) srcs.extend(glob.glob('test/harness/org/**/*.java', recursive=True)) exports = "" - if version > 7: + if version == 8: srcs.extend(glob.glob('test/harness/java8/**/*.java', recursive=True)) if version > 8: srcs.extend(glob.glob('test/harness/java9/**/*.java', recursive=True)) From a582302334e7614999032f8b8cab4fa0f1f9c861 Mon Sep 17 00:00:00 2001 From: "Karl E. Nelson" Date: Mon, 10 Oct 2022 09:47:05 -0700 Subject: [PATCH 5/8] Multiversion --- native/common/jp_exception.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/native/common/jp_exception.cpp b/native/common/jp_exception.cpp index 0e7fd3272..170cf89a4 100644 --- a/native/common/jp_exception.cpp +++ b/native/common/jp_exception.cpp @@ -519,7 +519,11 @@ PyObject *tb_create( return NULL; // Create a traceback +#if PY_VERSION_HEX<0x03110000 + JPPyObject lasti = JPPyObject::claim(PyLong_FromLong(pframe->f_lasti)); +#else JPPyObject lasti = JPPyObject::claim(PyLong_FromLong(PyFrame_GetLasti(pframe))); +#endif JPPyObject linenuma = JPPyObject::claim(PyLong_FromLong(linenum)); JPPyObject tuple = JPPyObject::call(PyTuple_Pack(4, Py_None, frame.get(), lasti.get(), linenuma.get())); JPPyObject traceback = JPPyObject::accept(PyObject_Call((PyObject*) &PyTraceBack_Type, tuple.get(), NULL)); From 2918afef51263fbe2907ec8cd65e998e8090a4a1 Mon Sep 17 00:00:00 2001 From: "Karl E. Nelson" Date: Mon, 10 Oct 2022 09:48:01 -0700 Subject: [PATCH 6/8] Documentation --- doc/CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/CHANGELOG.rst b/doc/CHANGELOG.rst index 9bd32fa55..af7b42585 100644 --- a/doc/CHANGELOG.rst +++ b/doc/CHANGELOG.rst @@ -8,6 +8,8 @@ Latest Changes: - Fixed issue with startJVM changing locale settings. + - Changes to support Python 3.11 + - **1.4.0 - 2022-05-14** - Support for all different buffer type conversions. From e8f6a09864b5165a2836aed1a54706a9b28f31b0 Mon Sep 17 00:00:00 2001 From: "Karl E. Nelson" Date: Mon, 17 Oct 2022 06:47:28 -0700 Subject: [PATCH 7/8] Fixes for review --- native/common/jp_exception.cpp | 6 ------ native/python/pyjp_value.cpp | 4 +--- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/native/common/jp_exception.cpp b/native/common/jp_exception.cpp index 170cf89a4..ddff7ce50 100644 --- a/native/common/jp_exception.cpp +++ b/native/common/jp_exception.cpp @@ -508,12 +508,6 @@ PyObject *tb_create( PyFrameObject *pframe = PyFrame_New(state, (PyCodeObject*) code.get(), dict, NULL); JPPyObject frame = JPPyObject::accept((PyObject*)pframe); - // Swap the back pointer - //PyObject* old = frame->f_back; - //frame->f_back = last_traceback->tb_frame; - //Py_XDECREF(old); - //Py_XINCREF(last_traceback->tb_frame); - // If we don't get the frame object there is no point if (frame.get() == NULL) return NULL; diff --git a/native/python/pyjp_value.cpp b/native/python/pyjp_value.cpp index e9ac16637..93dba9505 100644 --- a/native/python/pyjp_value.cpp +++ b/native/python/pyjp_value.cpp @@ -23,8 +23,6 @@ extern "C" { #endif -extern PyObject * _PyObject_GC_Malloc(size_t basicsize); - /** * Allocate a new Python object with a slot for Java. * @@ -49,7 +47,7 @@ PyObject* PyJPValue_alloc(PyTypeObject* type, Py_ssize_t nitems ) if (PyType_IS_GC(type)) { // Horrible kludge because python lacks an API for allocating a GC type with extra memory - // The privor method _PyObject_GC_Alloc is no longer visible, so we are forced to allocate + // The private method _PyObject_GC_Alloc is no longer visible, so we are forced to allocate // a different type with the extra memory and then hot swap the type to the real one. PyTypeObject type2; type2.tp_basicsize = size; From 64223ad96e94136f7a5b8ea63ff386f1c86b1840 Mon Sep 17 00:00:00 2001 From: "Karl E. Nelson" Date: Mon, 17 Oct 2022 09:17:11 -0700 Subject: [PATCH 8/8] For review --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e428e0e2..034f6637c 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,8 @@ packages=['jpype', 'jpype._pyinstaller'], package_dir={'jpype': 'jpype', }, package_data={'jpype': ['*.pyi']}, - install_requires=['typing_extensions ; python_version< "3.8"'], + install_requires=['typing_extensions ; python_version< "3.8"', + 'packaging ; python_version< "3.10"'], tests_require=['pytest'], extras_require={ 'tests': [