diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 491132d..eaa1bb3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: fail-fast: true matrix: os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.8','3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - name: Check out mdal-python @@ -80,7 +80,7 @@ jobs: - name: Publish package - test uses: pypa/gh-action-pypi-publish@release/v1 - if: github.event_name != 'release' + if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true with: user: __token__ password: ${{ secrets.PYPI_TEST_TOKEN }} diff --git a/.gitignore b/.gitignore index 36c6e18..7958219 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,5 @@ mdal.egg-info/ mdal-python.code-workspace mdal-pypi/LICENSE mdal-pypi/README.rst +test/results_3di.nc + diff --git a/CHANGES.rst b/CHANGES.rst index 3d43ef5..0b2123a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,13 +1,16 @@ Changes -------------------------------------------------------------------------------- -2.0.0 +1.0.2 ----- -UNDER DEVELOPMENT +- fix memory leaks and inconsistencies around the Datagroup object (#11) + +1.0.1 +----- + +- Add the PyPI package -- fix memory leaks and inconsistencies around the Datagroup object -- remove Open3D integration to separate package 1.0.0 ----- diff --git a/mdal/DatasetGroup.cpp b/mdal/DatasetGroup.cpp index e1a571f..d4b2583 100644 --- a/mdal/DatasetGroup.cpp +++ b/mdal/DatasetGroup.cpp @@ -122,8 +122,14 @@ PyArrayObject* Data::getDataAsDouble(int index) MDAL_SetStatus(MDAL_LogLevel::Error, MDAL_Status::Err_FailToWriteToDisk, "Could not import numpy.core.multiarray."); return (PyArrayObject*)defaultArray(); } + + MDAL_DatasetH datasetH = MDAL_G_dataset(m_data, index); + if ( MDAL_LastStatus() != MDAL_Status::None) + { + return (PyArrayObject*)defaultArray(); + } - npy_intp valueCount = (npy_intp)MDAL_D_valueCount( MDAL_G_dataset(m_data, index)); + npy_intp valueCount = (npy_intp)MDAL_D_valueCount( datasetH); int dims; MDAL_DataType type; if (MDAL_G_hasScalarData(m_data)) @@ -190,15 +196,16 @@ PyArrayObject* Data::getDataAsDouble(int index) } Py_XDECREF(dict); + Py_XDECREF(titles); + Py_XDECREF(formats); + Py_XDECREF(m_dataset); // This is a valueCount array. - PyArrayObject* dataset = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, + m_dataset = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &valueCount, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); double* buffer = new double[dims * 1024]; size_t count = 0; - int bufs = std::ceil(valueCount/1024); - if (bufs == 0) bufs = 1; int indexStart = 0; int next = 1024; @@ -207,7 +214,7 @@ PyArrayObject* Data::getDataAsDouble(int index) int remain = valueCount - indexStart; if (remain < 1024) next = remain; if (remain <= 0 ) break; - count = MDAL_D_data(MDAL_G_dataset(m_data, index), indexStart, next , type, buffer); + count = MDAL_D_data(datasetH, indexStart, next , type, buffer); if (count != next) { delete [] buffer; @@ -216,7 +223,7 @@ PyArrayObject* Data::getDataAsDouble(int index) int idx = 0; for (int i = 0; i < count; i++) { - char* p = (char *)PyArray_GETPTR1(dataset, indexStart + i); + char* p = (char *)PyArray_GETPTR1(m_dataset, indexStart + i); for (int l =0; l < dims; l++) { @@ -230,7 +237,7 @@ PyArrayObject* Data::getDataAsDouble(int index) } delete [] buffer; - return dataset; + return m_dataset; } PyArrayObject* Data::getDataAsVolumeIndex(int index) @@ -244,8 +251,13 @@ PyArrayObject* Data::getDataAsVolumeIndex(int index) MDAL_SetStatus(MDAL_LogLevel::Error, MDAL_Status::Err_FailToWriteToDisk, "Could not import numpy.core.multiarray."); return (PyArrayObject*)defaultArray(); } + + if ( MDAL_LastStatus() != MDAL_Status::None) + { + return (PyArrayObject*)defaultArray(); + } - npy_intp valueCount = (npy_intp)MDAL_M_faceCount( MDAL_G_mesh(m_data)); + npy_intp valueCount = (npy_intp)MDAL_M_faceCount( MDAL_G_mesh(m_data) ); PyObject* dict = PyDict_New(); PyObject* formats = PyList_New(1); @@ -265,15 +277,16 @@ PyArrayObject* Data::getDataAsVolumeIndex(int index) } Py_XDECREF(dict); + Py_XDECREF(titles); + Py_XDECREF(formats); + Py_XDECREF(m_dataset); // This is a dsCount x valueCount array. - PyArrayObject* dataset = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, + m_dataset = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &valueCount, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); int* buffer = new int[1024]; size_t count = 0; - int bufs = std::ceil(valueCount/1024); - if (bufs == 0) bufs = 1; int indexStart = 0; int next = 1024; @@ -282,7 +295,7 @@ PyArrayObject* Data::getDataAsVolumeIndex(int index) int remain = valueCount - indexStart; if (remain < 1024) next = remain; if (remain <= 0 ) break; - count = MDAL_D_data(MDAL_G_dataset(m_data, index), indexStart, next , MDAL_DataType::FACE_INDEX_TO_VOLUME_INDEX_INTEGER, buffer); + count = MDAL_D_data( MDAL_G_dataset(m_data, index), indexStart, next , MDAL_DataType::FACE_INDEX_TO_VOLUME_INDEX_INTEGER, buffer); if (count != next) { delete [] buffer; @@ -291,7 +304,7 @@ PyArrayObject* Data::getDataAsVolumeIndex(int index) int idx = 0; for (int i = 0; i < count; i++) { - char* p = (char *)PyArray_GETPTR1(dataset, indexStart + i); + char* p = (char *)PyArray_GETPTR1(m_dataset, indexStart + i); uint32_t val = (uint32_t)buffer[idx]; idx++; std::memcpy(p, &val, 4); @@ -300,12 +313,12 @@ PyArrayObject* Data::getDataAsVolumeIndex(int index) } delete [] buffer; - return dataset; + return m_dataset; } PyArrayObject* Data::getDataAsLevelCount(int index) { - if (! m_data) + if (! m_data) return (PyArrayObject*)defaultArray(); MDAL_ResetStatus(); @@ -314,6 +327,12 @@ PyArrayObject* Data::getDataAsLevelCount(int index) MDAL_SetStatus(MDAL_LogLevel::Error, MDAL_Status::Err_FailToWriteToDisk, "Could not import numpy.core.multiarray."); return (PyArrayObject*)defaultArray(); } + + MDAL_DatasetH datasetH = MDAL_G_dataset(m_data, index); + if ( MDAL_LastStatus() != MDAL_Status::None) + { + return (PyArrayObject*)defaultArray(); + } npy_intp valueCount = (npy_intp)MDAL_M_faceCount( MDAL_G_mesh(m_data)); @@ -335,15 +354,16 @@ PyArrayObject* Data::getDataAsLevelCount(int index) } Py_XDECREF(dict); + Py_XDECREF(titles); + Py_XDECREF(formats); + Py_XDECREF(m_dataset); // This is a dsCount x valueCount array. - PyArrayObject* dataset = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, + m_dataset = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &valueCount, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); int* buffer = new int[1024]; size_t count = 0; - int bufs = std::ceil(valueCount/1024); - if (bufs == 0) bufs = 1; int indexStart = 0; int next = 1024; @@ -352,7 +372,7 @@ PyArrayObject* Data::getDataAsLevelCount(int index) int remain = valueCount - indexStart; if (remain < 1024) next = remain; if (remain <= 0 ) break; - count = MDAL_D_data(MDAL_G_dataset(m_data, index), indexStart, next , MDAL_DataType::VERTICAL_LEVEL_COUNT_INTEGER, buffer); + count = MDAL_D_data( datasetH, indexStart, next , MDAL_DataType::VERTICAL_LEVEL_COUNT_INTEGER, buffer); if (count != next) { @@ -362,7 +382,7 @@ PyArrayObject* Data::getDataAsLevelCount(int index) int idx = 0; for (int i = 0; i < count; i++) { - char* p = (char *)PyArray_GETPTR1(dataset, indexStart + i); + char* p = (char *)PyArray_GETPTR1(m_dataset, indexStart + i); uint32_t val = (uint32_t)buffer[idx]; idx++; std::memcpy(p, &val, 4); @@ -371,7 +391,7 @@ PyArrayObject* Data::getDataAsLevelCount(int index) } delete [] buffer; - return dataset; + return m_dataset; } PyArrayObject* Data::getDataAsLevelValue(int index) @@ -385,6 +405,12 @@ PyArrayObject* Data::getDataAsLevelValue(int index) MDAL_SetStatus(MDAL_LogLevel::Error, MDAL_Status::Err_FailToWriteToDisk, "Could not import numpy.core.multiarray."); return (PyArrayObject*)defaultArray(); } + + MDAL_DatasetH datasetH = MDAL_G_dataset(m_data, index); + if ( MDAL_LastStatus() != MDAL_Status::None) + { + return (PyArrayObject*)defaultArray(); + } npy_intp valueCount = (npy_intp)MDAL_M_faceCount( MDAL_G_mesh(m_data)) + (npy_intp)MDAL_D_valueCount( MDAL_G_dataset(m_data, index)); @@ -406,15 +432,16 @@ PyArrayObject* Data::getDataAsLevelValue(int index) } Py_XDECREF(dict); + Py_XDECREF(titles); + Py_XDECREF(formats); + Py_XDECREF(m_dataset); // This is a dsCount x valueCount array. - PyArrayObject* dataset = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, + m_dataset = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &valueCount, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); double* buffer = new double[1024]; size_t count = 0; - int bufs = std::ceil(valueCount/1024); - if (bufs == 0) bufs = 1; int indexStart = 0; int next = 1024; @@ -423,7 +450,7 @@ PyArrayObject* Data::getDataAsLevelValue(int index) int remain = valueCount - indexStart; if (remain < 1024) next = remain; if (remain <= 0 ) break; - count = MDAL_D_data(MDAL_G_dataset(m_data, index), indexStart, next , MDAL_DataType::VERTICAL_LEVEL_DOUBLE, buffer); + count = MDAL_D_data( datasetH, indexStart, next , MDAL_DataType::VERTICAL_LEVEL_DOUBLE, buffer); if (count != next) { @@ -433,7 +460,7 @@ PyArrayObject* Data::getDataAsLevelValue(int index) int idx = 0; for (int i = 0; i < count; i++) { - char* p = (char *)PyArray_GETPTR1(dataset, indexStart + i); + char* p = (char *)PyArray_GETPTR1(m_dataset, indexStart + i); double val = buffer[idx]; idx++; std::memcpy(p, &val, 8); @@ -441,7 +468,7 @@ PyArrayObject* Data::getDataAsLevelValue(int index) indexStart += count; } delete [] buffer; - return dataset; + return m_dataset; } MDAL_Status Data::setDataAsDouble(PyArrayObject* data, double time) diff --git a/mdal/PyMesh.cpp b/mdal/PyMesh.cpp index 2f4a051..1605819 100644 --- a/mdal/PyMesh.cpp +++ b/mdal/PyMesh.cpp @@ -225,7 +225,9 @@ PyArrayObject *Mesh::getFaces() if (PyArray_DescrConverter(dict, &dtype) == NPY_FAIL) {} // throw pdal_error("Unable to build numpy dtype"); - Py_XDECREF(dict); + Py_XDECREF(dict); + Py_XDECREF(titles); + Py_XDECREF(formats); npy_intp size = (npy_intp)faceCount(); @@ -292,7 +294,9 @@ PyArrayObject *Mesh::getEdges() if (PyArray_DescrConverter(dict, &dtype) == NPY_FAIL) {} // throw pdal_error("Unable to build numpy dtype"); - Py_XDECREF(dict); + Py_XDECREF(dict); + Py_XDECREF(titles); + Py_XDECREF(formats); npy_intp size = (npy_intp)edgeCount(); @@ -450,8 +454,9 @@ bool Mesh::addFaces(PyArrayObject* faces, long int count){ PyObject *names_dict = dtype->fields; PyObject *names = PyDict_Keys(names_dict); PyObject *values = PyDict_Values(names_dict); - if (!names || !values) {} + if (!names || !values) { //throw pdal_error("Bad field specification in numpy array."); + } if (numFields< 3) {} //throw pdal_error("Faces" diff --git a/mdal/__init__.py b/mdal/__init__.py index 0f903fd..e74613e 100644 --- a/mdal/__init__.py +++ b/mdal/__init__.py @@ -11,7 +11,7 @@ from mdal.transform import MDAL_transform -__version__ = '1.0.1' +__version__ = '1.0.2' class Info(object): diff --git a/mdal/libmdalpython.pyx b/mdal/libmdalpython.pyx index efb57ac..bc13f36 100644 --- a/mdal/libmdalpython.pyx +++ b/mdal/libmdalpython.pyx @@ -228,12 +228,13 @@ cdef class Datasource: def __cinit__(self, uri: Path, driver_name : str = None): self.uri = str(uri) - if type(uri) != Path: - path = Path(uri) + if type(uri) == str: + uri = Path(uri) + print(uri) if driver_name: self.driver_name = driver_name return - if path.is_file(): + if uri.is_file(): try: meshes = self.meshes if len(meshes) > 0: @@ -241,7 +242,7 @@ cdef class Datasource: return except Exception: pass - ex = path.suffix + ex = uri.suffix for driver in drivers(): if ex in driver.filters: self.driver_name = driver.name diff --git a/mdal_python.egg-info/PKG-INFO b/mdal_python.egg-info/PKG-INFO index e7e2dc1..7cc6be3 100644 --- a/mdal_python.egg-info/PKG-INFO +++ b/mdal_python.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: mdal-python -Version: 1.0.0 +Version: 1.0.2 Summary: Mesh data processing Home-page: https://www.mdal.xyz Author: Paul Harwood @@ -21,6 +21,8 @@ Classifier: Programming Language :: Python :: 3.11 Classifier: Topic :: Scientific/Engineering :: GIS Description-Content-Type: text/x-rst License-File: LICENSE +Requires-Dist: numpy +Requires-Dist: meshio ================================================================================ MDAL Python Integration @@ -28,6 +30,9 @@ MDAL Python Integration .. image:: https://img.shields.io/conda/vn/conda-forge/mdal-python.svg :target: https://anaconda.org/conda-forge/mdal-python + +.. image:: https://badge.fury.io/py/mdal-python.svg + :target: https://badge.fury.io/py/mdal-python Basics ------ @@ -65,6 +70,17 @@ MDAL Python support is installable via Conda: conda install -c conda-forge mdal-python +PyPI +............................................................................... + +MDAL Python support can be installed using `pip` + +.. code-block:: + + pip install mdal-python + +This will ONLY work if there is a valid and working installation of MDAL on the device and accessible through the device library search path. + GitHub ................................................................................ diff --git a/mdal_python.egg-info/SOURCES.txt b/mdal_python.egg-info/SOURCES.txt index db71b87..bc9d9a1 100644 --- a/mdal_python.egg-info/SOURCES.txt +++ b/mdal_python.egg-info/SOURCES.txt @@ -2,17 +2,20 @@ LICENSE README.rst pyproject.toml setup.py -_skbuild/macosx-12.0-x86_64-3.10/cmake-install/mdal/__init__.py -_skbuild/macosx-12.0-x86_64-3.10/cmake-install/mdal/conf.py -_skbuild/macosx-12.0-x86_64-3.10/cmake-install/mdal/libmdalpython.cpython-310-darwin.so -_skbuild/macosx-12.0-x86_64-3.10/cmake-install/mdal/transform.py +_skbuild/win-amd64-3.11/cmake-install/mdal/__init__.py +_skbuild/win-amd64-3.11/cmake-install/mdal/conf.py +_skbuild/win-amd64-3.11/cmake-install/mdal/libmdalpython.cp311-win_amd64.pyd +_skbuild/win-amd64-3.11/cmake-install/mdal/transform.py +_skbuild/win-amd64-3.12/cmake-install/mdal/__init__.py +_skbuild/win-amd64-3.12/cmake-install/mdal/conf.py +_skbuild/win-amd64-3.12/cmake-install/mdal/libmdalpython.cp312-win_amd64.pyd +_skbuild/win-amd64-3.12/cmake-install/mdal/transform.py mdal/__init__.py mdal/conf.py mdal/transform.py mdal_python.egg-info/PKG-INFO mdal_python.egg-info/SOURCES.txt mdal_python.egg-info/dependency_links.txt -mdal_python.egg-info/not-zip-safe mdal_python.egg-info/requires.txt mdal_python.egg-info/top_level.txt test/test.py diff --git a/mdal_python.egg-info/not-zip-safe b/mdal_python.egg-info/not-zip-safe deleted file mode 100644 index 8b13789..0000000 --- a/mdal_python.egg-info/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/setup.py b/setup.py index 3ef6942..41a7015 100644 --- a/setup.py +++ b/setup.py @@ -72,10 +72,10 @@ 'Intended Audience :: Science/Research', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Scientific/Engineering :: GIS', ], diff --git a/test/data/save_test.nc b/test/data/save_test.nc index 1015011..148ab89 100644 Binary files a/test/data/save_test.nc and b/test/data/save_test.nc differ diff --git a/test/leak.py b/test/leak.py new file mode 100644 index 0000000..d86a726 --- /dev/null +++ b/test/leak.py @@ -0,0 +1,18 @@ +from mdal import Datasource +from pathlib import Path +import gc + +mypath = "/Users/paulharwood/Downloads/results_3di.nc" +ds = Datasource(mypath) + +for i in range(0,100): + print(i) + with ds.load() as mesh: + print("mesh loaded") + group = mesh.group(3) + print("group Loaded") + for j in range(0, group.dataset_count): + data = group.data(j) + print(data) + # make very sure the variable gets taken out of memory: + del(data) \ No newline at end of file diff --git a/test/leak2.py b/test/leak2.py new file mode 100644 index 0000000..772c2a2 --- /dev/null +++ b/test/leak2.py @@ -0,0 +1,14 @@ +from mdal import Datasource +from pathlib import Path +import gc + +mypath = "/Users/paulharwood/Library/CloudStorage/GoogleDrive-runette@gmail.com/My Drive/Mesh Project/gpr_new.ply" +ds = Datasource(mypath) + +with ds.load() as mesh: + print("mesh loaded") + for i in range(0,1000): + print(i) + group = mesh.group(3) + data = group.data(0) + print(data) \ No newline at end of file diff --git a/test/leak3.py b/test/leak3.py new file mode 100644 index 0000000..ce7a4df --- /dev/null +++ b/test/leak3.py @@ -0,0 +1,12 @@ +from mdal import Datasource +from pathlib import Path +import gc + +mypath = "/Users/paulharwood/Library/CloudStorage/GoogleDrive-runette@gmail.com/My Drive/Mesh Project/gpr_new.ply" +ds = Datasource(mypath) + +with ds.load() as mesh: + print("mesh loaded") + for i in range(0,100000): + print(i) + vertices = mesh.vertices \ No newline at end of file diff --git a/test/save_test.ply b/test/save_test.ply index 9fcf360..8f48c6c 100644 --- a/test/save_test.ply +++ b/test/save_test.ply @@ -1,9 +1,13 @@ ply format ascii 1.0 +obj_info Generated by MDAL +comment comment1: Generated by PDAL +comment crs: EPSG:4326 +comment test: value element vertex 38487 -property double X -property double Y -property double Z +property double x +property double y +property double z property double red property double green property double blue diff --git a/test/save_test_2.ply b/test/save_test_2.ply index c41c556..bbd84c0 100644 --- a/test/save_test_2.ply +++ b/test/save_test_2.ply @@ -1,9 +1,14 @@ ply format ascii 1.0 +obj_info Generated by MDAL +comment comment1: test file for MDAL +comment crs: +proj=tmerc +comment comment2: test1 +comment comment3: test2 element vertex 5 -property double X -property double Y -property double Z +property double x +property double y +property double z property double red property double green property double blue diff --git a/test/test.py b/test/test.py index fcc5d1a..5c5ffa9 100644 --- a/test/test.py +++ b/test/test.py @@ -37,7 +37,7 @@ from mdal import Datasource, Info, last_status, PyMesh, drivers, MDAL_DataLocation, MDAL_transform import numpy as np -import open3d as o3d + print(f"MDAL Version: {Info.version}") print(f"MDAL Driver Count :{Info.driver_count}") @@ -194,21 +194,4 @@ print(f"{mio2}") -with Datasource("data/ply/test_mesh.ply").load() as mesh: - tm = MDAL_transform.to_triangle_mesh(mesh) - print(tm) - tm2 = o3d.io.read_triangle_mesh("data/ply/test_mesh.ply") - tmc = np.asarray(tm.vertex_colors) - tmc2 = np.asarray(tm2.vertex_colors) - for i in range(len(tmc)): - value = tmc[i] - tmc2[i] - if not (value == [0, 0, 0]).all(): - print(value) - break - -with Datasource("test_vol.ply").load() as mesh: - pc = MDAL_transform.to_point_cloud(mesh.group(1)) - print(pc) - - print("all finished !") diff --git a/test/test.vtk b/test/test.vtk index d6fa98c..8d22899 100644 Binary files a/test/test.vtk and b/test/test.vtk differ diff --git a/test/test_vol.ply b/test/test_vol.ply index cdf2712..2055b55 100644 Binary files a/test/test_vol.ply and b/test/test_vol.ply differ