From 99418df5804b146e73e00dd6590ba2c23e0d416d Mon Sep 17 00:00:00 2001 From: Jianfeng Mao <4297243+jmao-denver@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:33:14 -0700 Subject: [PATCH] Add jpy.byte_buffer() function (#112) * Add jpy.byte_buffer() function * Fix memory leak when error occurs * Fix potential memory leak with PyBuffer_release * Update src/main/c/jpy_jbyte_buffer.c Co-authored-by: Chip Kent <5250374+chipkent@users.noreply.github.com> * Auto perform buffer->ByteBuffer conv for args * Refactor code in preparation for varargs support * No support of varargs of Python buffer objects * Tidy up a bit * Fix a type-checking error in dealloc * Remove auto-conversion * Remove unused code and improve docstring * Add and use JByteBuffer_Check for cstr/dstr * Naming change * Remove unnecessary decl --------- Co-authored-by: Chip Kent <5250374+chipkent@users.noreply.github.com> --- src/main/c/jpy_conv.c | 1 - src/main/c/jpy_jbyte_buffer.c | 24 +++++++++ src/main/c/jpy_jbyte_buffer.h | 44 ++++++++++++++++ src/main/c/jpy_jobj.c | 37 +++++++++++-- src/main/c/jpy_jobj.h | 2 + src/main/c/jpy_jtype.c | 2 +- src/main/c/jpy_module.c | 99 ++++++++++++++++++++++++++++++++++- src/main/c/jpy_module.h | 4 ++ 8 files changed, 206 insertions(+), 7 deletions(-) create mode 100644 src/main/c/jpy_jbyte_buffer.c create mode 100644 src/main/c/jpy_jbyte_buffer.h diff --git a/src/main/c/jpy_conv.c b/src/main/c/jpy_conv.c index 1af0c8d4..e7d64534 100644 --- a/src/main/c/jpy_conv.c +++ b/src/main/c/jpy_conv.c @@ -293,4 +293,3 @@ int JPy_AsJString(JNIEnv* jenv, PyObject* arg, jstring* stringRef) return 0; } - diff --git a/src/main/c/jpy_jbyte_buffer.c b/src/main/c/jpy_jbyte_buffer.c new file mode 100644 index 00000000..f443646f --- /dev/null +++ b/src/main/c/jpy_jbyte_buffer.c @@ -0,0 +1,24 @@ +/* + * Copyright 2023 JPY-CONSORTIUM Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "jpy_module.h" +#include "jpy_diag.h" +#include "jpy_jarray.h" +#include "jpy_jbyte_buffer.h" + +/* + * This file for now is just a place-holder for future JPy_JByteBufferObj specific functions. + */ \ No newline at end of file diff --git a/src/main/c/jpy_jbyte_buffer.h b/src/main/c/jpy_jbyte_buffer.h new file mode 100644 index 00000000..6b44cb3d --- /dev/null +++ b/src/main/c/jpy_jbyte_buffer.h @@ -0,0 +1,44 @@ +/* + * Copyright 2023 JPY-CONSORTIUM Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JPY_JBYTE_BUFFER_H +#define JPY_JBYTE_BUFFER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "jpy_compat.h" + +/** + * The Java ByteBuffer representation in Python. + * + * IMPORTANT: JPy_JByteBufferObj must only differ from the JPy_JObj structure by the 'pyBuffer' member + * since we use the same basic type, name JPy_JType for it. DON'T ever change member positions! + * @see JPy_JObj + */ +typedef struct JPy_JByteBufferObj +{ + PyObject_HEAD + jobject objectRef; + Py_buffer *pyBuffer; +} +JPy_JByteBufferObj; + +#ifdef __cplusplus +} /* extern "C" */ +#endif +#endif /* !JPY_JBYTE_BUFFER_H */ diff --git a/src/main/c/jpy_jobj.c b/src/main/c/jpy_jobj.c index 579bdce9..e7fcf66c 100644 --- a/src/main/c/jpy_jobj.c +++ b/src/main/c/jpy_jobj.c @@ -20,6 +20,7 @@ #include "jpy_module.h" #include "jpy_diag.h" #include "jpy_jarray.h" +#include "jpy_jbyte_buffer.h" #include "jpy_jtype.h" #include "jpy_jobj.h" #include "jpy_jmethod.h" @@ -63,9 +64,15 @@ PyObject* JObj_FromType(JNIEnv* jenv, JPy_JType* type, jobject objectRef) array = (JPy_JArray*) obj; array->bufferExportCount = 0; array->buf = NULL; + } else if (JByteBuffer_Check(type)) { + JPy_JByteBufferObj *byteBuffer; + + byteBuffer = (JPy_JByteBufferObj *) obj; + byteBuffer->pyBuffer = NULL; } - // we check the type translations dictionary for a callable for this java type name, + +// we check the type translations dictionary for a callable for this java type name, // and apply the returned callable to the wrapped object callable = PyDict_GetItemString(JPy_Type_Translations, type->javaName); if (callable != NULL) { @@ -181,8 +188,14 @@ void JObj_dealloc(JPy_JObj* self) if (array->buf != NULL) { JArray_ReleaseJavaArrayElements(array, array->javaType); } - - } + } else if (JByteBuffer_Check(jtype)) { + JPy_JByteBufferObj *byteBuffer; + byteBuffer = (JPy_JByteBufferObj *) self; + if (byteBuffer->pyBuffer != NULL) { + PyBuffer_Release(byteBuffer->pyBuffer); + PyMem_Free(byteBuffer->pyBuffer); + } + } jenv = JPy_GetJNIEnv(); if (jenv != NULL) { @@ -727,7 +740,13 @@ int JType_InitSlots(JPy_JType* type) //Py_SET_TYPE(type, &JType_Type); //Py_SET_SIZE(type, sizeof (JPy_JType)); - typeObj->tp_basicsize = isPrimitiveArray ? sizeof (JPy_JArray) : sizeof (JPy_JObj); + if (isPrimitiveArray) { + typeObj->tp_basicsize = sizeof(JPy_JArray); + } else if (JByteBuffer_Check(type)) { + typeObj->tp_basicsize = sizeof(JPy_JByteBufferObj); + } else { + typeObj->tp_basicsize = sizeof(JPy_JObj); + } typeObj->tp_itemsize = 0; typeObj->tp_base = type->superType != NULL ? JTYPE_AS_PYTYPE(type->superType) : &JType_Type; //typeObj->tp_base = (PyTypeObject*) type->superType; @@ -822,3 +841,13 @@ int JType_Check(PyObject* arg) return PyType_Check(arg) && JPY_IS_JTYPE(arg); } +int JByteBuffer_Check(JPy_JType* type) { + while (type != NULL) { + if (type == JPy_JByteBuffer || strcmp(type->javaName, "java.nio.ByteBuffer") == 0) { + JPy_DIAG_PRINT(JPy_DIAG_F_TYPE, "JByteBuffer_Check: java ByteBuffer or its sub-type (%s) found.\n", type->javaName); + return -1; + } + type = type->superType; + } + return 0; +} diff --git a/src/main/c/jpy_jobj.h b/src/main/c/jpy_jobj.h index 3454f307..5b740df3 100644 --- a/src/main/c/jpy_jobj.h +++ b/src/main/c/jpy_jobj.h @@ -40,6 +40,8 @@ JPy_JObj; int JObj_Check(PyObject* arg); +int JByteBuffer_Check(JPy_JType* type); + PyObject* JObj_New(JNIEnv* jenv, jobject objectRef); PyObject* JObj_FromType(JNIEnv* jenv, JPy_JType* type, jobject objectRef); diff --git a/src/main/c/jpy_jtype.c b/src/main/c/jpy_jtype.c index 04843171..5fff4e11 100644 --- a/src/main/c/jpy_jtype.c +++ b/src/main/c/jpy_jtype.c @@ -23,6 +23,7 @@ #include "jpy_jfield.h" #include "jpy_jmethod.h" #include "jpy_jobj.h" +#include "jpy_jbyte_buffer.h" #include "jpy_conv.h" #include "jpy_compat.h" @@ -1659,7 +1660,6 @@ int JType_ConvertPyArgToJPyObjectArg(JNIEnv* jenv, JPy_ParamDescriptor* paramDes return JType_CreateJavaPyObject(jenv, JPy_JPyObject, pyArg, &value->l); } - int JType_MatchPyArgAsJPyObjectParam(JNIEnv* jenv, JPy_ParamDescriptor* paramDescriptor, PyObject* pyArg) { // We can always turn a python object into a PyObject diff --git a/src/main/c/jpy_module.c b/src/main/c/jpy_module.c index e8716fca..ccf8563f 100644 --- a/src/main/c/jpy_module.c +++ b/src/main/c/jpy_module.c @@ -26,6 +26,7 @@ #include "jpy_jobj.h" #include "jpy_conv.h" #include "jpy_compat.h" +#include "jpy_jbyte_buffer.h" #include @@ -38,6 +39,7 @@ PyObject* JPy_destroy_jvm(PyObject* self, PyObject* args); PyObject* JPy_get_type(PyObject* self, PyObject* args, PyObject* kwds); PyObject* JPy_cast(PyObject* self, PyObject* args); PyObject* JPy_array(PyObject* self, PyObject* args); +PyObject* JPy_byte_buffer(PyObject* self, PyObject* args); static PyMethodDef JPy_Functions[] = { @@ -63,6 +65,11 @@ static PyMethodDef JPy_Functions[] = { "array(name, init) - Return a new Java array of given Java type (type name or type object) and initializer (array length or sequence). " "Possible primitive types are 'boolean', 'byte', 'char', 'short', 'int', 'long', 'float', and 'double'."}, + {"byte_buffer", JPy_byte_buffer, METH_VARARGS, + "byte_buffer(obj) - Return a new Java direct ByteBuffer sharing the same underlying, contiguous buffer of obj via its implemented Buffer Protocol. The resulting PYObject must live " + "longer than the Java object to ensure the underlying data remains valid. In most cases, this means that java functions called in this manner must not keep any references" + " to the ByteBuffer"}, + {NULL, NULL, 0, NULL} /*Sentinel*/ }; @@ -126,7 +133,7 @@ JPy_JType* JPy_JPyObject = NULL; JPy_JType* JPy_JPyModule = NULL; JPy_JType* JPy_JThrowable = NULL; JPy_JType* JPy_JStackTraceElement = NULL; - +JPy_JType* JPy_JByteBuffer = NULL; // java.lang.Comparable jclass JPy_Comparable_JClass = NULL; @@ -228,6 +235,8 @@ jclass JPy_Void_JClass = NULL; jclass JPy_String_JClass = NULL; jclass JPy_PyObject_JClass = NULL; jclass JPy_PyDictWrapper_JClass = NULL; +jclass JPy_ByteBuffer_JClass = NULL; +jmethodID JPy_ByteBuffer_AsReadOnlyBuffer_MID = NULL; jmethodID JPy_PyObject_GetPointer_MID = NULL; jmethodID JPy_PyObject_UnwrapProxy_SMID = NULL; @@ -660,6 +669,83 @@ PyObject* JPy_array(PyObject* self, PyObject* args) JPy_FRAME(PyObject*, NULL, JPy_array_internal(jenv, self, args), 16) } +PyObject* JType_CreateJavaByteBufferObj(JNIEnv* jenv, PyObject* pyObj) +{ + jobject byteBufferRef, tmpByteBufferRef; + Py_buffer *pyBuffer; + PyObject *newPyObj; + JPy_JByteBufferObj* byteBuffer; + + pyBuffer = (Py_buffer *)PyMem_Malloc(sizeof(Py_buffer)); + if (pyBuffer == NULL) { + PyErr_NoMemory(); + return NULL; + } + + if (PyObject_GetBuffer(pyObj, pyBuffer, PyBUF_SIMPLE | PyBUF_C_CONTIGUOUS) != 0) { + PyErr_SetString(PyExc_ValueError, "JType_CreateJavaByteBufferObj: the Python object failed to return a contiguous buffer."); + PyMem_Free(pyBuffer); + return NULL; + } + + tmpByteBufferRef = (*jenv)->NewDirectByteBuffer(jenv, pyBuffer->buf, pyBuffer->len); + if (tmpByteBufferRef == NULL) { + PyBuffer_Release(pyBuffer); + PyMem_Free(pyBuffer); + PyErr_NoMemory(); + return NULL; + } + + byteBufferRef = (*jenv)->CallObjectMethod(jenv, tmpByteBufferRef, JPy_ByteBuffer_AsReadOnlyBuffer_MID); + if (byteBufferRef == NULL) { + PyBuffer_Release(pyBuffer); + PyMem_Free(pyBuffer); + JPy_DELETE_LOCAL_REF(tmpByteBufferRef); + PyErr_SetString(PyExc_RuntimeError, "jpy: internal error: failed to create a read-only direct ByteBuffer instance."); + return NULL; + } + JPy_DELETE_LOCAL_REF(tmpByteBufferRef); + + newPyObj = JObj_New(jenv, byteBufferRef); + if (newPyObj == NULL) { + PyErr_SetString(PyExc_RuntimeError, "jpy: internal error: failed to create a byteBuffer instance."); + PyBuffer_Release(pyBuffer); + PyMem_Free(pyBuffer); + JPy_DELETE_LOCAL_REF(byteBufferRef); + return NULL; + } + JPy_DELETE_LOCAL_REF(byteBufferRef); + + byteBuffer = (JPy_JByteBufferObj *) newPyObj; + byteBuffer->pyBuffer = pyBuffer; + return (PyObject *)byteBuffer; +} + +PyObject* JPy_byte_buffer_internal(JNIEnv* jenv, PyObject* self, PyObject* args) +{ + jobject byteBufferRef; + PyObject* pyObj; + PyObject* newPyObj; + Py_buffer *pyBuffer; + JPy_JByteBufferObj* byteBuffer; + + if (!PyArg_ParseTuple(args, "O:byte_buffer", &pyObj)) { + return NULL; + } + + if (PyObject_CheckBuffer(pyObj) == 0) { + PyErr_SetString(PyExc_ValueError, "byte_buffer: argument 1 must be a Python object that supports the buffer protocol."); + return NULL; + } + + return JType_CreateJavaByteBufferObj(jenv, pyObj); +} + +PyObject* JPy_byte_buffer(PyObject* self, PyObject* args) +{ + JPy_FRAME(PyObject*, NULL, JPy_byte_buffer_internal(jenv, self, args), 16) +} + JPy_JType* JPy_GetNonObjectJType(JNIEnv* jenv, jclass classRef) { jclass primClassRef; @@ -927,6 +1013,9 @@ int JPy_InitGlobalVars(JNIEnv* jenv) DEFINE_CLASS(JPy_String_JClass, "java/lang/String"); DEFINE_CLASS(JPy_Throwable_JClass, "java/lang/Throwable"); DEFINE_CLASS(JPy_StackTraceElement_JClass, "java/lang/StackTraceElement"); + DEFINE_CLASS(JPy_ByteBuffer_JClass, "java/nio/ByteBuffer"); + DEFINE_METHOD(JPy_ByteBuffer_AsReadOnlyBuffer_MID, JPy_ByteBuffer_JClass, "asReadOnlyBuffer", "()Ljava/nio/ByteBuffer;"); + // Non-Object types: Primitive types and void. DEFINE_NON_OBJECT_TYPE(JPy_JBoolean, JPy_Boolean_JClass); @@ -953,6 +1042,8 @@ int JPy_InitGlobalVars(JNIEnv* jenv) DEFINE_OBJECT_TYPE(JPy_JDoubleObj, JPy_Double_JClass); // Other objects. DEFINE_OBJECT_TYPE(JPy_JString, JPy_String_JClass); + DEFINE_OBJECT_TYPE(JPy_JByteBuffer, JPy_ByteBuffer_JClass); + DEFINE_OBJECT_TYPE(JPy_JThrowable, JPy_Throwable_JClass); DEFINE_OBJECT_TYPE(JPy_JStackTraceElement, JPy_StackTraceElement_JClass); DEFINE_METHOD(JPy_Throwable_getCause_MID, JPy_Throwable_JClass, "getCause", "()Ljava/lang/Throwable;"); @@ -994,6 +1085,7 @@ void JPy_ClearGlobalVars(JNIEnv* jenv) (*jenv)->DeleteGlobalRef(jenv, JPy_Number_JClass); (*jenv)->DeleteGlobalRef(jenv, JPy_Void_JClass); (*jenv)->DeleteGlobalRef(jenv, JPy_String_JClass); + (*jenv)->DeleteGlobalRef(jenv, JPy_ByteBuffer_JClass); } JPy_Comparable_JClass = NULL; @@ -1014,6 +1106,7 @@ void JPy_ClearGlobalVars(JNIEnv* jenv) JPy_Number_JClass = NULL; JPy_Void_JClass = NULL; JPy_String_JClass = NULL; + JPy_ByteBuffer_JClass = NULL; JPy_Object_ToString_MID = NULL; JPy_Object_HashCode_MID = NULL; @@ -1051,6 +1144,7 @@ void JPy_ClearGlobalVars(JNIEnv* jenv) JPy_Number_DoubleValue_MID = NULL; JPy_PyObject_GetPointer_MID = NULL; JPy_PyObject_UnwrapProxy_SMID = NULL; + JPy_ByteBuffer_AsReadOnlyBuffer_MID = NULL; JPy_XDECREF(JPy_JBoolean); JPy_XDECREF(JPy_JChar); @@ -1061,6 +1155,8 @@ void JPy_ClearGlobalVars(JNIEnv* jenv) JPy_XDECREF(JPy_JFloat); JPy_XDECREF(JPy_JDouble); JPy_XDECREF(JPy_JVoid); + JPy_XDECREF(JPy_JString); + JPy_XDECREF(JPy_JByteBuffer); JPy_XDECREF(JPy_JBooleanObj); JPy_XDECREF(JPy_JCharacterObj); JPy_XDECREF(JPy_JByteObj); @@ -1082,6 +1178,7 @@ void JPy_ClearGlobalVars(JNIEnv* jenv) JPy_JDouble = NULL; JPy_JVoid = NULL; JPy_JString = NULL; + JPy_JByteBuffer = NULL; JPy_JBooleanObj = NULL; JPy_JCharacterObj = NULL; JPy_JByteObj = NULL; diff --git a/src/main/c/jpy_module.h b/src/main/c/jpy_module.h index 36b48f31..1607681f 100644 --- a/src/main/c/jpy_module.h +++ b/src/main/c/jpy_module.h @@ -156,6 +156,7 @@ extern struct JPy_JType* JPy_JClass; extern struct JPy_JType* JPy_JString; extern struct JPy_JType* JPy_JPyObject; extern struct JPy_JType* JPy_JPyModule; +extern struct JPy_JType* JPy_JByteBuffer; // java.lang.Comparable extern jclass JPy_Comparable_JClass; @@ -250,6 +251,9 @@ extern jmethodID JPy_Number_DoubleValue_MID; extern jclass JPy_String_JClass; extern jclass JPy_Void_JClass; +extern jclass JPy_ByteBuffer_JClass; +extern jmethodID JPy_ByteBuffer_AsReadOnlyBuffer_MID; + extern jclass JPy_PyObject_JClass; extern jmethodID JPy_PyObject_GetPointer_MID; extern jmethodID JPy_PyObject_UnwrapProxy_SMID;