From 5e31b65dd55866ca2491875a5b1b782673e0f558 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Sun, 28 Apr 2024 09:36:51 -0500 Subject: [PATCH] Set Numpy floor to 1.22 and NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION (#167) * Set Numpy floor to 1.22 and NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION * set PY_ARRAY_UNIQUE_SYMBOL to PDAL_ARRAY_API and only import it in extension initialization. Guard against using the extension against any PDAL verisions < 2.6 * debug PDAL_DRIVER_PATH issues * bug in --print-plugin-path * only one os.path.sep * concat PDAL_DRIVER_PATH with PDAL_PLUGIN_PATH --- .github/workflows/build.yml | 8 ++++++-- CMakeLists.txt | 4 +++- pyproject.toml | 4 ++-- src/pdal/PyArray.cpp | 3 --- src/pdal/PyArray.hpp | 9 +++++++++ src/pdal/PyPipeline.cpp | 16 +--------------- src/pdal/PyPipeline.hpp | 7 +++++++ src/pdal/StreamableExecutor.cpp | 10 ++++++---- src/pdal/__main__.py | 2 +- src/pdal/libpdalpython.cpp | 18 ++++++++++++++++++ 10 files changed, 53 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65f01ff3..42c82a78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,8 +57,12 @@ jobs: - name: Test run: | - export PDAL_DRIVER_PATH=$(python -m pdal --pdal-plugin-path) - echo $PDAL_DRIVER_PATH + export PDAL_DRIVER_PATH=$(python -m pdal --pdal-driver-path) + export PDAL_PLUGIN_PATH=$(python -m pdal --pdal-plugin-path) + echo "PDAL_DRIVER_PATH $PDAL_DRIVER_PATH" + echo "PDAL_PLUGIN_PATH $PDAL_PLUGIN_PATH" + export PDAL_DRIVER_PATH=$PDAL_PLUGIN_PATH:$PDAL_DRIVER_PATH + python -m pdal pdal --drivers --debug py.test -v test/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c2860ed..e22cd1a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.11.0) -project(pdal-python VERSION) +project(pdal-python VERSION ${SKBUILD_PROJECT_VERSION} + DESCRIPTION "PDAL Python bindings" + HOMEPAGE_URL "https://github.com/PDAL/Python") set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/pyproject.toml b/pyproject.toml index 419a8b08..eca080ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ - "numpy" + "numpy >= 1.22" ] dynamic = ["version"] @@ -48,7 +48,7 @@ repository = "https://github.com/PDAL/Python" changelog = "https://github.com/PDAL/python/blob/main/README.rst" [build-system] -requires = ["scikit-build-core >= 0.9", "numpy", "pybind11[global]"] +requires = ["scikit-build-core >= 0.9", "numpy >= 1.22", "pybind11[global]"] build-backend = "scikit_build_core.build" diff --git a/src/pdal/PyArray.cpp b/src/pdal/PyArray.cpp index 04c424bd..66b6bd45 100644 --- a/src/pdal/PyArray.cpp +++ b/src/pdal/PyArray.cpp @@ -92,9 +92,6 @@ std::string toString(PyObject *pname) Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) { - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); - Py_XINCREF(array); PyArray_Descr *dtype = PyArray_DTYPE(m_array); diff --git a/src/pdal/PyArray.hpp b/src/pdal/PyArray.hpp index f0401f00..e2da961f 100644 --- a/src/pdal/PyArray.hpp +++ b/src/pdal/PyArray.hpp @@ -35,8 +35,17 @@ #pragma once #include + +#define NPY_TARGET_VERSION NPY_1_22_API_VERSION +#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION + +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + #include + #include +#include #include #include diff --git a/src/pdal/PyPipeline.cpp b/src/pdal/PyPipeline.cpp index 35859f06..85ea028a 100644 --- a/src/pdal/PyPipeline.cpp +++ b/src/pdal/PyPipeline.cpp @@ -211,16 +211,7 @@ std::string PipelineExecutor::getQuickInfo() const void PipelineExecutor::addArrayReaders(std::vector> arrays) { - // Make the symbols in pdal_base global so that they're accessible - // to PDAL plugins. Python dlopen's this extension with RTLD_LOCAL, - // which means that without this, symbols in libpdal_base aren't available - // for resolution of symbols on future runtime linking. This is an issue - // on Alpine and other Linux variants that don't use UNIQUE symbols - // for C++ template statics only. Without this, you end up with multiple - // copies of template statics. -#ifndef _WIN32 - ::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL); -#endif + if (arrays.empty()) return; @@ -320,8 +311,6 @@ PyObject* buildNumpyDescriptor(PointLayoutPtr layout) PyArrayObject* viewToNumpyArray(PointViewPtr view) { - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); PyObject* dtype_dict = buildNumpyDescriptor(view->layout()); PyArray_Descr *dtype = nullptr; @@ -344,9 +333,6 @@ PyArrayObject* viewToNumpyArray(PointViewPtr view) PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh) { - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); - // Build up a numpy dtype dictionary // // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', diff --git a/src/pdal/PyPipeline.hpp b/src/pdal/PyPipeline.hpp index c4e316c6..c32abcfe 100644 --- a/src/pdal/PyPipeline.hpp +++ b/src/pdal/PyPipeline.hpp @@ -35,6 +35,13 @@ #pragma once #include + +#define NPY_TARGET_VERSION NPY_1_22_API_VERSION +#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION + +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + #include namespace pdal diff --git a/src/pdal/StreamableExecutor.cpp b/src/pdal/StreamableExecutor.cpp index ef9950e2..9f5b4b8b 100644 --- a/src/pdal/StreamableExecutor.cpp +++ b/src/pdal/StreamableExecutor.cpp @@ -35,6 +35,9 @@ #include "PyPipeline.hpp" #include "StreamableExecutor.hpp" +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + #include #include @@ -67,8 +70,7 @@ void PythonPointTable::finalize() // create dtype auto gil = PyGILState_Ensure(); - if (_import_array() < 0) - std::cerr << "Could not import array!\n"; + PyObject *dtype_dict = buildNumpyDescriptor(&m_layout); if (PyArray_DescrConverter(dtype_dict, &m_dtype) == NPY_FAIL) throw pdal_error("Unable to create numpy dtype"); @@ -102,8 +104,8 @@ void PythonPointTable::py_resizeArray(point_count_t np) { if (src_idx != dest_idx) { - PyObject* src_item = PyArray_GETITEM(m_curArray, PyArray_GETPTR1(m_curArray, src_idx)); - PyArray_SETITEM(m_curArray, PyArray_GETPTR1(m_curArray, dest_idx), src_item); + PyObject* src_item = PyArray_GETITEM(m_curArray, (const char*) PyArray_GETPTR1(m_curArray, src_idx)); + PyArray_SETITEM(m_curArray, (char*) PyArray_GETPTR1(m_curArray, dest_idx), src_item); Py_XDECREF(src_item); } dest_idx++; diff --git a/src/pdal/__main__.py b/src/pdal/__main__.py index f45f3151..7fa7d272 100644 --- a/src/pdal/__main__.py +++ b/src/pdal/__main__.py @@ -23,7 +23,7 @@ def print_driver_path(args): print (os.environ['PDAL_DRIVER_PATH']) def print_plugin_path(args): - purelib = sysconfig.get_paths()["purelib"]+os.path.sep+"pdal" + purelib = sysconfig.get_paths()["purelib"] if sys.platform == "linux" or sys.platform == "linux2": suffix = 'so' diff --git a/src/pdal/libpdalpython.cpp b/src/pdal/libpdalpython.cpp index 7bd526a0..229f928d 100644 --- a/src/pdal/libpdalpython.cpp +++ b/src/pdal/libpdalpython.cpp @@ -7,6 +7,13 @@ #include #include +#define NPY_TARGET_VERSION NPY_1_22_API_VERSION +#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION + +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + +#include + #include "PyArray.hpp" #include "PyDimension.hpp" #include "PyPipeline.hpp" @@ -283,8 +290,12 @@ namespace pdal { int _loglevel; }; + + PYBIND11_MODULE(libpdalpython, m) { + _import_array(); + py::class_(m, "PipelineIterator") .def("__iter__", [](PipelineIterator &it) -> PipelineIterator& { return it; }) .def("__next__", &PipelineIterator::executeNext) @@ -294,6 +305,7 @@ namespace pdal { .def_property_readonly("pipeline", &PipelineIterator::getPipeline) .def_property_readonly("metadata", &PipelineIterator::getMetadata); + py::class_(m, "Pipeline") .def(py::init<>()) .def("execute", &Pipeline::execute) @@ -319,6 +331,12 @@ namespace pdal { m.def("getDimensions", &getDimensions); m.def("infer_reader_driver", &getReaderDriver); m.def("infer_writer_driver", &getWriterDriver); + + if (pdal::Config::versionMajor() < 2) + throw pybind11::import_error("PDAL version must be >= 2.6"); + + if (pdal::Config::versionMajor() == 2 && pdal::Config::versionMinor() < 6) + throw pybind11::import_error("PDAL version must be >= 2.6"); }; }; // namespace pdal