Merge branch 'master' into tidyup_virtual_override_mess

partial merge, jp_exception still to do

Author: marscher

.azure/build.yml

@@ -62,6 +62,9 @@ jobs:
       linux-3.10:
         imageName: "ubuntu-latest"
         python.version: '3.10'
+      linux-3.11:
+        imageName: "ubuntu-latest"
+        python.version: '3.11'
       windows-3.7:
         imageName: "windows-2019"
         python.version: '3.7'
@@ -75,6 +78,9 @@ jobs:
       windows-3.10:
         imageName: "windows-2019"
         python.version: '3.10'
+      windows-3.11:
+        imageName: "windows-2019"
+        python.version: '3.11'
       mac-3.9:
         imageName: "macos-11"
         python.version: '3.9'

.azure/release.yml

@@ -68,6 +68,9 @@ stages:
         Python310:
           python.version: '3.10'
           python.architecture: 'x64'
+        Python311:
+          python.version: '3.11'
+          python.architecture: 'x64'
     pool:
       vmImage: "windows-2019"
     steps:
@@ -96,6 +99,8 @@ stages:
           python.version: '3.9'
         Python310:
           python.version: '3.10'
+        Python311:
+          python.version: '3.11'
     pool:
       vmImage: "macos-11"
     steps:

.azure/scripts/osx-python.sh

@@ -19,6 +19,10 @@ case $PYTHON_VERSION in
   FULL_VERSION=3.10.4
   INSTALLER_NAME=python-$FULL_VERSION-macos11.pkg
   ;;
+3.11)
+  FULL_VERSION=3.11.0
+  INSTALLER_NAME=python-$FULL_VERSION-macos11.pkg
+  ;;
 esac
 
 URL=https://www.python.org/ftp/python/$FULL_VERSION/$INSTALLER_NAME

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.4.1_dev0
+current_version = 1.4.2_dev0
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\_(?P<release>[a-z]+)(?P<build>\d+))?

MANIFEST.in

@@ -1,7 +1,8 @@
 recursive-include native *.h *.xml *.java *.jar *.class
 recursive-include examples *
 include *.rst doc/* LICENSE
-# include everything under test/
+# include test/
 graft test
+prune test/classes/*
 graft setupext
 global-exclude *.pyc

doc/CHANGELOG.rst

@@ -4,7 +4,21 @@ Changelog
 This changelog *only* contains changes from the *first* pypi release (0.5.4.3) onwards.
 
 Latest Changes:
-- **1.4.1_dev0 - 2022-05-14**
+- **1.4.2_dev0 - 2022-10-26**
+
+  - Fixed direct byte buffers not reporting nbytes correctly when cast to memoryview.
+
+- **1.4.1 - 2022-10-26**
+  
+  - Fixed issue with startJVM changing locale settings.
+
+  - Changes to support Python 3.11
+
+  - Fix truncation of strings on null when using convert strings.
+
+  - Replaced distutil with packaging
+
+
 - **1.4.0 - 2022-05-14**
 
   - Support for all different buffer type conversions.

doc/userguide.rst

@@ -92,7 +92,7 @@ set the class path, start the JVM, remove all the type declarations, and you are
 
    # Copy in the patterns from the guide to replace the example code
    db = Database("our_records")
-   with db.connect() as DatabaseConnection:
+   with db.connect() as c:
        c.runQuery()
        while c.hasRecords():
            record = db.nextRecord()

jpype/__init__.py

@@ -51,7 +51,7 @@
 __all__.extend(_jcustomizer.__all__)
 __all__.extend(_gui.__all__)
 
-__version__ = "1.4.1_dev0"
+__version__ = "1.4.2_dev0"
 __version_info__ = __version__.split('.')
 
 

jpype/_core.py

@@ -218,9 +218,22 @@ def startJVM(*args, **kwargs):
                         % (','.join([str(i) for i in kwargs])))
 
     try:
+        import locale
+        # Gather a list of locale settings that Java may override (excluding LC_ALL)
+        categories = [getattr(locale, i) for i in dir(locale) if i.startswith('LC_') and i != 'LC_ALL']
+        # Keep the current locale settings, else Java will replace them.
+        prior = [locale.getlocale(i) for i in categories]
+        # Start the JVM
         _jpype.startup(jvmpath, tuple(args),
                        ignoreUnrecognized, convertStrings, interrupt)
+        # Collect required resources for operation
         initializeResources()
+        # Restore locale
+        for i, j in zip(categories, prior):
+            try:
+                locale.setlocale(i, j)
+            except locale.Error:
+                pass
     except RuntimeError as ex:
         source = str(ex)
         if "UnsupportedClassVersion" in source:

jpype/_jproxy.py

@@ -59,6 +59,9 @@ def _createJProxyDeferred(cls, *intf):
     @JOverride notation on methods evaluated at first
     instantiation.
     """
+    if not isinstance(cls, type):
+        raise TypeError("JImplements only applies to types, not %s" % (type(cls)))
+
     def new(tp, *args, **kwargs):
         # Attach a __jpype_interfaces__ attribute to this class if
         # one doesn't already exist.
@@ -77,6 +80,9 @@ def _createJProxy(cls, *intf):
     """ (internal) Create a proxy from a Python class with
     @JOverride notation on methods evaluated at declaration.
     """
+    if not isinstance(cls, type):
+        raise TypeError("JImplements only applies to types, not %s" % (type(cls)))
+
     actualIntf = _prepareInterfaces(cls, intf)
 
     def new(tp, *args, **kwargs):

jpype/_jvmfinder.py

@@ -320,10 +320,10 @@ def _javahome_binary(self):
         """
         import platform
         import subprocess
-        from distutils.version import StrictVersion
+        from packaging.version import Version
 
-        current = StrictVersion(platform.mac_ver()[0][:4])
-        if current >= StrictVersion('10.6') and current < StrictVersion('10.9'):
+        current = Version(platform.mac_ver()[0][:4])
+        if current >= Version('10.6') and current < Version('10.9'):
             return subprocess.check_output(
                 ['/usr/libexec/java_home']).strip()
 

native/common/jp_buffer.cpp

@@ -37,7 +37,8 @@ JPBuffer::JPBuffer(const JPValue &value)
 	m_Buffer.readonly = frame.isBufferReadOnly(m_Object.get());
 	m_Buffer.shape = &m_Capacity;
 	m_Buffer.strides = &m_Buffer.itemsize;
-	m_Buffer.suboffsets = nullptr;
+	m_Buffer.suboffsets = 0;
+	m_Buffer.len = m_Buffer.itemsize * m_Capacity;
 	JP_TRACE_OUT;  // GCOVR_EXCL_LINE
 }
 

native/common/jp_context.cpp

@@ -215,14 +215,10 @@ void JPContext::initializeResources(JNIEnv* env, bool interrupt)
 
 	if (!m_Embedded)
 	{
-		JPPyObject import = JPPyObject::call(PyImport_AddModule("importlib.util"));
+		JPPyObject import = JPPyObject::use(PyImport_AddModule("importlib.util"));
 		JPPyObject jpype = JPPyObject::call(PyObject_CallMethod(import.get(), "find_spec", "s", "_jpype"));
 		JPPyObject origin = JPPyObject::call(PyObject_GetAttrString(jpype.get(), "origin"));
 		val[2].l = frame.fromStringUTF8(JPPyString::asStringUTF8(origin.get()));
-		import.incref();  // The documentation specifies that PyImport_AddModule must return a 
-	                  // new reference, but that is not happening in Python 3.10
-	                  // so we are triggering a gc assertion failure.  To prevent
-	                  // the failure manually up the reference counter here.
 	}
 	m_JavaContext = JPObjectRef(frame, frame.CallStaticObjectMethodA(contextClass, startMethod, val));
 

native/common/jp_exception.cpp

@@ -72,11 +72,11 @@ JPypeException::JPypeException(const JPypeException &ex) noexcept
 
 JPypeException& JPypeException::operator = (const JPypeException& ex)
 {
-    if(this == &ex)
-    {
-        return *this;
-    }
-    m_Context = ex.m_Context;
+	if(this == &ex)
+	{
+		return *this;
+	}
+	m_Context = ex.m_Context;
 	m_Type = ex.m_Type;
 	m_Trace = ex.m_Trace;
 	m_Throwable = ex.m_Throwable;
@@ -88,9 +88,40 @@ JPypeException& JPypeException::operator = (const JPypeException& ex)
 void JPypeException::from(const JPStackInfo& info)
 {
 	JP_TRACE("EXCEPTION FROM: ", info.getFile(), info.getLine());
-	m_Trace.emplace_back(info);
+	m_Trace.push_back(info);
 }
 
+// Okay from this point on we have to suit up in full Kevlar because
+// this code must handle every conceivable and still reach a resolution.
+// Exceptions may be throws during initialization where only a fraction
+// of the resources are available, during the middle of normal operation,
+// or worst of all as the system is being yanked out from under us during
+// shutdown.  Each and every one of these cases must be considered.
+// Further each and every function called here must be hardened similarly
+// or they will become the weak link. And remember it is not paranoia if
+// they are actually out to get you.
+//
+// Onward my friends to victory or a glorious segfault!
+/*
+string JPypeException::getMessage()
+{
+	JP_TRACE_IN("JPypeException::getMessage");
+	// Must be bullet proof
+	try
+	{
+		stringstream str;
+		str << m_Message << endl;
+		JP_TRACE(str.str());
+		return str.str();
+		// GCOVR_EXCL_START
+	} catch (...)
+	{
+		return "error during get message";
+	}
+	JP_TRACE_OUT;
+	// GCOVR_EXCL_STOP
+}*/
+
 bool isJavaThrowable(PyObject* exceptionClass)
 {
 	JPClass* cls = PyJPClass_getJPClass(exceptionClass);
@@ -169,7 +200,7 @@ void JPypeException::convertJavaToPython()
 	}
 	// GCOVR_EXCL_STOP
 
-	auto *type = (PyObject*) Py_TYPE(pyvalue.get());
+	PyObject *type = (PyObject*) Py_TYPE(pyvalue.get());
 	Py_INCREF(type);
 
 	// Add cause to the exception
@@ -449,83 +480,56 @@ void JPypeException::toJava(JPContext *context)
 	JP_TRACE_OUT; // GCOVR_EXCL_LINE
 }
 
-PyTracebackObject *tb_create(
-		PyTracebackObject *last_traceback,
+PyObject *tb_create(
+		PyObject *last_traceback,
 		PyObject *dict,
 		const char* filename,
 		const char* funcname,
 		int linenum)
 {
 	// Create a code for this frame. (ref count is 1)
-	PyCodeObject *code = PyCode_NewEmpty(filename, funcname, linenum);
+	JPPyObject* code = JPPyObject::accept((PyObject*)PyCode_NewEmpty(filename, funcname, linenum));
 
 	// If we don't get the code object there is no point
 	if (code == nullptr)
 		return nullptr;
 
-	// This is a bit of a kludge.  Python lacks a way to directly create
-	// a frame from a code object except when creating from the threadstate.
-	//
-	// In reviewing Python implementation, I find that the only element accessed
-	// in the thread state was the previous frame to link to.  Because frame 
-	// objects change a lot between different Python versions, trying to 
-	// replicate the actions of setting up a frame is difficult to keep portable.
-	//
-	// Python 3.10 introduces the additional requirement that the global
-	// dictionary supplied must have a __builtins__.  We can do this once
-	// when create the module.
-	//
-	// If instead we create a thread state and point the field it needs to the
-	// previous frame we create the frames using the defined API.  Much more 
-	// portable, but we have to create a big (uninitialized object) each time we
-	// want to pass in the previous frame.
-	PyThreadState state;
-	if (last_traceback != nullptr)
-		state.frame = last_traceback->tb_frame;
-	else
-		state.frame = nullptr;
-
 	// Create a frame for the traceback.
-	PyFrameObject *frame = PyFrame_New(&state, code, dict, nullptr);
-	
-	// frame just borrows the reference rather than claiming it
-	// so we need to get rid of the extra reference here.
-	Py_DECREF(code);
-	
+	PyThreadState *state = PyThreadState_GET();
+	PyFrameObject *pframe = PyFrame_New(state, (PyCodeObject*) code.get(), dict, NULL);
+	JPPyObject frame = JPPyObject::accept((PyObject*)pframe);
+
 	// If we don't get the frame object there is no point
-	if (frame == nullptr)
+	if (frame.get() == nullptr)
 		return nullptr;
 
 	// Create a traceback
-	auto *traceback = (PyTracebackObject*) 
-		        PyObject_GC_New(PyTracebackObject, &PyTraceBack_Type);
+#if PY_MINOR_VERSION<11
+	JPPyObject lasti = JPPyObject::claim(PyLong_FromLong(pframe->f_lasti));
+#else
+	JPPyObject lasti = JPPyObject::claim(PyLong_FromLong(PyFrame_GetLasti(pframe)));
+#endif
+	JPPyObject linenuma = JPPyObject::claim(PyLong_FromLong(linenum));
+	JPPyObject tuple = JPPyObject::call(PyTuple_Pack(4, Py_None, frame.get(), lasti.get(), linenuma.get()));
+	JPPyObject traceback = JPPyObject::accept(PyObject_Call((PyObject*) &PyTraceBack_Type, tuple.get(), NULL));
 
 	// We could fail in process
-	if (traceback == nullptr)
+	if (traceback.get() == nullptr)
 	{
 		Py_DECREF(frame);
 		return nullptr;
 	}
 
-	// Set the fields
-	traceback->tb_frame = frame; // Steal the reference from frame
-	traceback->tb_lasti = frame->f_lasti;
-	traceback->tb_lineno = linenum;
-	Py_XINCREF(last_traceback);
-	traceback->tb_next = last_traceback;
-
-	// Allow GC on the object
-	PyObject_GC_Track(traceback);
-	return traceback;
+	return traceback.keep();
 }
 
 PyObject* PyTrace_FromJPStackTrace(JPStackTrace& trace)
 {
-	PyTracebackObject *last_traceback = nullptr;
+	PyObject *last_traceback = nullptr;
 	PyObject *dict = PyModule_GetDict(PyJPModule);
-	for (auto & iter : trace)
+	for (auto& iter : trace)
 	{
-		last_traceback = tb_create(last_traceback, dict,  iter.getFile(),
+		last_traceback = tb_create(last_traceback, dict, iter.getFile(),
 				iter.getFunction(), iter.getLine());
 	}
 	if (last_traceback == nullptr)
@@ -544,8 +548,8 @@ JPPyObject PyTrace_FromJavaException(JPJavaFrame& frame, jthrowable th, jthrowab
 		return {};
 
 	JNIEnv* env = frame.getEnv();
-	auto obj = (jobjectArray) env->CallObjectMethodA(context->getJavaContext(),
-			context->m_Context_GetStackFrameID, args);
+	jobjectArray obj = static_cast<jobjectArray>(env->CallObjectMethodA(context->getJavaContext(),
+			context->m_Context_GetStackFrameID, args));
 
 	// Eat any exceptions that were generated
 	if (env->ExceptionCheck() == JNI_TRUE)
@@ -559,8 +563,8 @@ JPPyObject PyTrace_FromJavaException(JPJavaFrame& frame, jthrowable th, jthrowab
 	{
 		string filename, method;
 		auto jclassname = static_cast<jstring>(frame.GetObjectArrayElement(obj, i));
-		auto jmethodname = (jstring) frame.GetObjectArrayElement(obj, i + 1);
-		auto jfilename = (jstring) frame.GetObjectArrayElement(obj, i + 2);
+		auto jmethodname = static_cast<jstring>(frame.GetObjectArrayElement(obj, i + 1));
+		auto jfilename = static_cast<jstring>(frame.GetObjectArrayElement(obj, i + 2));
 		if (jfilename != nullptr)
 			filename = frame.toStringUTF8(jfilename);
 		else

native/common/jp_stringtype.cpp

@@ -46,7 +46,7 @@ JPPyObject JPStringType::convertToPythonObject(JPJavaFrame& frame, jvalue val, b
 		if (context->getConvertStrings())
 		{
 			string str = frame.toStringUTF8((jstring) (val.l));
-			return JPPyObject::call(PyUnicode_FromString(str.c_str()));
+			return JPPyObject::call(PyUnicode_FromStringAndSize(str.c_str(), str.length()));
 		}
 	}