Merge pull request #1087 from Thrameos/py3.11

Fixes for 3.11

doc/CHANGELOG.rst

@@ -8,6 +8,8 @@ Latest Changes:
 
  - Fixed issue with startJVM changing locale settings.
 
+ - Changes to support Python 3.11
+
 - **1.4.0 - 2022-05-14**
 
  - Support for all different buffer type conversions.

jpype/_jvmfinder.py

@@ -320,10 +320,10 @@ def _javahome_binary(self):
  """
  import platform
  import subprocess
- from distutils.version import StrictVersion
+ from packaging.version import Version
 
- current = StrictVersion(platform.mac_ver()[0][:4])
- if current >= StrictVersion('10.6') and current < StrictVersion('10.9'):
+ current = Version(platform.mac_ver()[0][:4])
+ if current >= Version('10.6') and current < Version('10.9'):
  return subprocess.check_output(
  ['/usr/libexec/java_home']).strip()
 

native/common/jp_context.cpp

@@ -282,14 +282,10 @@ void JPContext::initializeResources(JNIEnv* env, bool interrupt)
 
  if (!m_Embedded)
  {
- JPPyObject import = JPPyObject::call(PyImport_AddModule("importlib.util"));
+ JPPyObject import = JPPyObject::use(PyImport_AddModule("importlib.util"));
  JPPyObject jpype = JPPyObject::call(PyObject_CallMethod(import.get(), "find_spec", "s", "_jpype"));
  JPPyObject origin = JPPyObject::call(PyObject_GetAttrString(jpype.get(), "origin"));
  val[2].l = frame.fromStringUTF8(JPPyString::asStringUTF8(origin.get()));
- import.incref(); // The documentation specifies that PyImport_AddModule must return a 
- // new reference, but that is not happening in Python 3.10
- // so we are triggering a gc assertion failure. To prevent
- // the failure manually up the reference counter here.
  }
  m_JavaContext = JPObjectRef(frame, frame.CallStaticObjectMethodA(contextClass, startMethod, val));
 

native/common/jp_exception.cpp

@@ -489,79 +489,51 @@ void JPypeException::toJava(JPContext *context)
  JP_TRACE_OUT; // GCOVR_EXCL_LINE
 }
 
-PyTracebackObject *tb_create(
- PyTracebackObject *last_traceback,
+PyObject *tb_create(
+ PyObject *last_traceback,
  PyObject *dict,
  const char* filename,
  const char* funcname,
  int linenum)
 {
  // Create a code for this frame. (ref count is 1)
- PyCodeObject *code = PyCode_NewEmpty(filename, funcname, linenum);
+ JPPyObject code = JPPyObject::accept((PyObject*)PyCode_NewEmpty(filename, funcname, linenum));
 
  // If we don't get the code object there is no point
- if (code == NULL)
+ if (code.get() == NULL)
  return NULL;
 
- // This is a bit of a kludge. Python lacks a way to directly create
- // a frame from a code object except when creating from the threadstate.
- //
- // In reviewing Python implementation, I find that the only element accessed
- // in the thread state was the previous frame to link to. Because frame 
- // objects change a lot between different Python versions, trying to 
- // replicate the actions of setting up a frame is difficult to keep portable.
- //
- // Python 3.10 introduces the additional requirement that the global
- // dictionary supplied must have a __builtins__. We can do this once
- // when create the module.
- //
- // If instead we create a thread state and point the field it needs to the
- // previous frame we create the frames using the defined API. Much more 
- // portable, but we have to create a big (uninitialized object) each time we
- // want to pass in the previous frame.
- PyThreadState state;
- if (last_traceback != NULL)
- state.frame = last_traceback->tb_frame;
- else
- state.frame = NULL;
-
  // Create a frame for the traceback.
- PyFrameObject *frame = PyFrame_New(&state, code, dict, NULL);
-
- // frame just borrows the reference rather than claiming it
- // so we need to get rid of the extra reference here.
- Py_DECREF(code);
-
+ PyThreadState *state = PyThreadState_GET();
+ PyFrameObject *pframe = PyFrame_New(state, (PyCodeObject*) code.get(), dict, NULL);
+ JPPyObject frame = JPPyObject::accept((PyObject*)pframe);
+
  // If we don't get the frame object there is no point
- if (frame == NULL)
+ if (frame.get() == NULL)
  return NULL;
 
  // Create a traceback
- PyTracebackObject *traceback = (PyTracebackObject*) 
- PyObject_GC_New(PyTracebackObject, &PyTraceBack_Type);
+#if PY_VERSION_HEX<0x03110000
+ JPPyObject lasti = JPPyObject::claim(PyLong_FromLong(pframe->f_lasti));
+#else
+ JPPyObject lasti = JPPyObject::claim(PyLong_FromLong(PyFrame_GetLasti(pframe)));
+#endif
+ JPPyObject linenuma = JPPyObject::claim(PyLong_FromLong(linenum));
+ JPPyObject tuple = JPPyObject::call(PyTuple_Pack(4, Py_None, frame.get(), lasti.get(), linenuma.get()));
+ JPPyObject traceback = JPPyObject::accept(PyObject_Call((PyObject*) &PyTraceBack_Type, tuple.get(), NULL));
 
  // We could fail in process
- if (traceback == NULL)
+ if (traceback.get() == NULL)
  {
- Py_DECREF(frame);
  return NULL;
  }
 
- // Set the fields
- traceback->tb_frame = frame; // Steal the reference from frame
- traceback->tb_lasti = frame->f_lasti;
- traceback->tb_lineno = linenum;
- Py_XINCREF(last_traceback);
- traceback->tb_next = last_traceback;
-
- // Allow GC on the object
- PyObject_GC_Track(traceback);
- return traceback;
+ return traceback.keep();
 }
 
 PyObject* PyTrace_FromJPStackTrace(JPStackTrace& trace)
 {
- PyTracebackObject *last_traceback = NULL;
+ PyObject *last_traceback = NULL;
  PyObject *dict = PyModule_GetDict(PyJPModule);
  for (JPStackTrace::iterator iter = trace.begin(); iter != trace.end(); ++iter)
  {
@@ -575,7 +547,7 @@ PyObject* PyTrace_FromJPStackTrace(JPStackTrace& trace)
 
 JPPyObject PyTrace_FromJavaException(JPJavaFrame& frame, jthrowable th, jthrowable prev)
 {
- PyTracebackObject *last_traceback = NULL;
+ PyObject *last_traceback = NULL;
  JPContext *context = frame.getContext();
  jvalue args[2];
  args[0].l = th;

native/python/pyjp_value.cpp

@@ -43,8 +43,28 @@ PyObject* PyJPValue_alloc(PyTypeObject* type, Py_ssize_t nitems )
  JP_PY_TRY("PyJPValue_alloc");
  // Modification from Python to add size elements
  const size_t size = _PyObject_VAR_SIZE(type, nitems + 1) + sizeof (JPValue);
- PyObject *obj = (PyType_IS_GC(type)) ? _PyObject_GC_Malloc(size)
- : (PyObject *) PyObject_MALLOC(size);
+ PyObject *obj = NULL;
+ if (PyType_IS_GC(type))
+ {
+ // Horrible kludge because python lacks an API for allocating a GC type with extra memory
+ // The private method _PyObject_GC_Alloc is no longer visible, so we are forced to allocate
+ // a different type with the extra memory and then hot swap the type to the real one.
+ PyTypeObject type2;
+ type2.tp_basicsize = size;
+ type2.tp_itemsize = 0;
+ type2.tp_name = NULL;
+ type2.tp_flags = type->tp_flags;
+ type2.tp_traverse = type->tp_traverse;
+
+ // Allocate the fake type
+ obj = PyObject_GC_New(PyObject, &type2);
+
+ // Note the object will be inited twice which should not leak. (fingers crossed)
+ }
+ else
+ {
+ obj = (PyObject*) PyObject_MALLOC(size);
+ }
  if (obj == NULL)
  return PyErr_NoMemory(); // GCOVR_EXCL_LINE
  memset(obj, 0, size);
@@ -305,4 +325,4 @@ bool PyJPValue_isSetJavaSlot(PyObject* self)
  return false; // GCOVR_EXCL_LINE
  JPValue* slot = (JPValue*) (((char*) self) + offset);
  return slot->getClass() != NULL;
-}
+}

setup.py

@@ -78,7 +78,8 @@
  packages=['jpype', 'jpype._pyinstaller'],
  package_dir={'jpype': 'jpype', },
  package_data={'jpype': ['*.pyi']},
- install_requires=['typing_extensions ; python_version< "3.8"'],
+ install_requires=['typing_extensions ; python_version< "3.8"',
+ 'packaging ; python_version< "3.10"'],
  tests_require=['pytest'],
  extras_require={
  'tests': [

setupext/test_java.py

@@ -53,7 +53,7 @@ def compileJava():
  srcs = glob.glob('test/harness/jpype/**/*.java', recursive=True)
  srcs.extend(glob.glob('test/harness/org/**/*.java', recursive=True))
  exports = ""
- if version > 7:
+ if version == 8:
  srcs.extend(glob.glob('test/harness/java8/**/*.java', recursive=True))
  if version > 8:
  srcs.extend(glob.glob('test/harness/java9/**/*.java', recursive=True))