Skip to content

Commit

Permalink
Add a libgdal-arrow-parquet package
Browse files Browse the repository at this point in the history
which contains the ogr_Arrow.so/dylib/dll and ogr_Parquet.so/dylib/dll
plugins

The libgdal-arrow-parquet build reuses the same build directory as
libgdal, hence we just to have to turn on the required dependency and
drivers in the build_arrow_parquet.sh/bat scripts.

Fixes #628

Also improve build time of the Python bindings by doing a minimal build
(relates to #662, but a better fix should be possible)
  • Loading branch information
rouault committed Nov 18, 2022
1 parent 0e48914 commit f16f19f
Show file tree
Hide file tree
Showing 10 changed files with 189 additions and 8 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
From eee1b439437788e9ecb65b600b8cd33d2012be32 Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Fri, 18 Nov 2022 02:09:18 +0100
Subject: [PATCH] Parquet: make sure that ArrowLayer destructor is available
(for plugin)

---
ogr/ogrsf_frmts/parquet/ogrparquetdriver.cpp | 1 +
1 file changed, 1 insertion(+)

diff --git a/ogr/ogrsf_frmts/parquet/ogrparquetdriver.cpp b/ogr/ogrsf_frmts/parquet/ogrparquetdriver.cpp
index ff112586ed..3303f999f4 100644
--- a/ogr/ogrsf_frmts/parquet/ogrparquetdriver.cpp
+++ b/ogr/ogrsf_frmts/parquet/ogrparquetdriver.cpp
@@ -35,6 +35,7 @@
#include "../arrow_common/ograrrowrandomaccessfile.h"
#include "../arrow_common/ograrrowwritablefile.h"
#include "../arrow_common/ograrrowdataset.hpp"
+#include "../arrow_common/ograrrowlayer.hpp" // for the destructor

/************************************************************************/
/* Identify() */
--
2.25.1

9 changes: 9 additions & 0 deletions recipe/bld.bat
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ if %vc% GTR 9 set MSVC_VER=1900
if %vc% LEQ 9 set MSVC_TS_VER=90
if %vc% GTR 9 set MSVC_TS_VER=140

REM Make sure to disable Arrow/Parquet dependencies for now, so they are only
REM used in build_arrow_parquet

cmake -G "Ninja" ^
"%CMAKE_ARGS%" ^
-DMSVC_VERSION="%MSVC_VER%" ^
Expand All @@ -27,6 +30,12 @@ cmake -G "Ninja" ^
-DKEA_LIBRARY:PATH="%LIBRARY_LIB%\libkea.lib" ^
-DGDAL_USE_MYSQL:BOOL=OFF ^
-DGDAL_USE_MSSQL_ODBC:BOOL=OFF ^
-DGDAL_USE_PARQUET=OFF ^
-DGDAL_USE_ARROW=OFF ^
-DGDAL_USE_ARROWDATASET=OFF ^
-DBUILD_PYTHON_BINDINGS:BOOL=OFF ^
-DBUILD_JAVA_BINDINGS:BOOL=OFF ^
-DBUILD_CSHARP_BINDINGS:BOOL=OFF ^
"%SRC_DIR%"

if errorlevel 1 exit /b 1
Expand Down
5 changes: 5 additions & 0 deletions recipe/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,18 @@ export CXXFLAGS="${CXXFLAGS} -std=c++17 -D_LIBCPP_DISABLE_AVAILABILITY"
mkdir build
cd build

# Make sure to disable Arrow/Parquet dependencies for now, so they are only
# used in build_arrow_parquet
cmake -G "Unix Makefiles" \
${CMAKE_ARGS} \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH=$PREFIX \
-DCMAKE_INSTALL_PREFIX=$PREFIX \
-DCMAKE_INSTALL_LIBDIR=lib \
-DBUILD_SHARED_LIBS=ON \
-DGDAL_USE_PARQUET=OFF \
-DGDAL_USE_ARROW=OFF \
-DGDAL_USE_ARROWDATASET=OFF \
-DBUILD_PYTHON_BINDINGS:BOOL=OFF \
-DBUILD_JAVA_BINDINGS:BOOL=OFF \
-DBUILD_CSHARP_BINDINGS:BOOL=OFF \
Expand Down
21 changes: 21 additions & 0 deletions recipe/build_arrow_parquet.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
cd build
if errorlevel 1 exit 1

REM We reuse the same build directory as libgdal, so we just to have to
REM turn on the required dependency and drivers
cmake -DGDAL_USE_PARQUET=ON ^
-DGDAL_USE_ARROW=ON ^
-DGDAL_USE_ARROWDATASET=ON ^
-DOGR_ENABLE_DRIVER_ARROW=ON ^
-DOGR_ENABLE_DRIVER_ARROW_PLUGIN=ON ^
-DOGR_ENABLE_DRIVER_PARQUET=ON ^
-DOGR_ENABLE_DRIVER_PARQUET_PLUGIN=ON ^
"%SRC_DIR%"

if errorlevel 1 exit /b 1

cmake --build . -j %CPU_COUNT% --verbose --config Release
if errorlevel 1 exit /b 1

cmake --build . --target install --config Release
if errorlevel 1 exit /b 1
19 changes: 19 additions & 0 deletions recipe/build_arrow_parquet.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/bin/bash

set -ex # Abort on error.

cd build

# We reuse the same build directory as libgdal, so we just to have to
# turn on the required dependency and drivers
cmake -DGDAL_USE_PARQUET=ON \
-DGDAL_USE_ARROW=ON \
-DGDAL_USE_ARROWDATASET=ON \
-DOGR_ENABLE_DRIVER_ARROW=ON \
-DOGR_ENABLE_DRIVER_ARROW_PLUGIN=ON \
-DOGR_ENABLE_DRIVER_PARQUET=ON \
-DOGR_ENABLE_DRIVER_PARQUET_PLUGIN=ON \
${SRC_DIR}

cmake --build . -j ${CPU_COUNT} --config Release
cmake --build . --target install
9 changes: 3 additions & 6 deletions recipe/install_python.bat
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,9 @@ cmake -G "Ninja" ^
-DBUILD_SHARED_LIBS=ON ^
-DPython_EXECUTABLE="%PYTHON%" ^
-DBUILD_PYTHON_BINDINGS:BOOL=ON ^
-DBUILD_JAVA_BINDINGS:BOOL=OFF ^
-DBUILD_CSHARP_BINDINGS:BOOL=OFF ^
-DGDAL_USE_KEA:BOOL=ON ^
-DKEA_INCLUDE_DIR:PATH="%LIBRARY_INC%" ^
-DKEA_LIBRARY:PATH="%LIBRARY_LIB%\libkea.lib" ^
-DGDAL_USE_MYSQL:BOOL=OFF ^
-DGDAL_USE_EXTERNAL_LIBS=OFF ^
-DGDAL_BUILD_OPTIONAL_DRIVERS:BOOL=OFF ^
-DOGR_BUILD_OPTIONAL_DRIVERS:BOOL=OFF ^
"%SRC_DIR%"

if errorlevel 1 exit /b 1
Expand Down
6 changes: 5 additions & 1 deletion recipe/install_python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ export CXXFLAGS="${CXXFLAGS} -std=c++17 -D_LIBCPP_DISABLE_AVAILABILITY"
mkdir pybuild_${PKG_HASH}
pushd pybuild_${PKG_HASH}

# Do a minimum build of libgdal
cmake -G "Unix Makefiles" \
${CMAKE_ARGS} \
-DCMAKE_BUILD_TYPE=Release \
Expand All @@ -21,6 +22,9 @@ cmake -G "Unix Makefiles" \
-DBUILD_PYTHON_BINDINGS:BOOL=ON \
-DBUILD_JAVA_BINDINGS:BOOL=OFF \
-DBUILD_CSHARP_BINDINGS:BOOL=OFF \
-DGDAL_USE_EXTERNAL_LIBS=OFF \
-DGDAL_BUILD_OPTIONAL_DRIVERS:BOOL=OFF \
-DOGR_BUILD_OPTIONAL_DRIVERS:BOOL=OFF \
${SRC_DIR} || (cat CMakeFiles/CMakeError.log;false)


Expand All @@ -32,7 +36,7 @@ requires = ["setuptools>=40.8.0", "wheel"]
build-backend = "setuptools.build_meta"
EOF

cmake --build . --target python_wheel
cmake --build . --target python_wheel -j ${CPU_COUNT}

$PYTHON -m pip install --no-deps --ignore-installed .

Expand Down
86 changes: 85 additions & 1 deletion recipe/meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@ source:
sha256: f7afa4aa8d32d0799e011a9f573c6a67e9471f78e70d3d0d0b45b45c8c0c1a94
patches:
- 0001-Improve-numpy-fixing.patch
- 0002-Parquet-make-sure-that-ArrowLayer-destructor-is-avai.patch

build:
number: 3
number: 4
skip_compile_pyc:
- "share/bash-completion/completions/*.py"

Expand Down Expand Up @@ -50,6 +51,7 @@ requirements:
- libxml2
- openjpeg
- openssl
- parquet-cpp
- pcre2
- poppler
- postgresql
Expand Down Expand Up @@ -142,6 +144,88 @@ outputs:
license: MIT
license_file: LICENSE.TXT

- name: libgdal-arrow-parquet
script: build_arrow_parquet.sh # [unix]
script: build_arrow_parquet.bat # [win]
requirements:
build:
- cmake
# ranlib used at install time
- {{ compiler('c') }}
# libstdc++ is needed in requirements/run
- {{ compiler('cxx') }}
- pkg-config # [not win]
- make # [unix]
- ninja # [win]
- sysroot_linux-64 2.17 # [linux64]
host:
- {{ pin_subpackage('libgdal', exact=True) }}
- blosc
- cfitsio
- curl
- expat
- freexl
- geos
- geotiff
- giflib # [not win]
- hdf4
- hdf5
- icu
- jpeg
- json-c # [not win]
- kealib
- libkml
- libnetcdf
- libpng
- libpq
- libspatialite
- libtiff
- libuuid # [linux]
- libwebp-base
- libxml2
- openjpeg
- openssl
- parquet-cpp
- pcre2
- poppler
- postgresql
- proj
- sqlite
- tiledb
- xerces-c
- xz
- zlib
- zstd
run:
- {{ pin_subpackage('libgdal', exact=True) }}
- arrow-cpp
- cfitsio
- geotiff
- giflib # [not win]
- json-c # [not win]
- libpq
- libspatialite
- libuuid # [linux]
- libwebp-base
- parquet-cpp
- poppler
- postgresql
- proj
- tiledb
- xz
- zstd
test:
files:
- test_arrow_parquet.sh # [unix]
- test_arrow_parquet.bat # [win]
commands:
- sh test_arrow_parquet.sh # [unix]
- test_arrow_parquet.bat # [win]
about:
summary: Arrow and Parquet drivers for the Geospatial Data Abstraction Library (GDAL)
license: MIT
license_file: LICENSE.TXT

- name: gdal
script: install_python.sh # [unix]
script: install_python.bat # [win]
Expand Down
5 changes: 5 additions & 0 deletions recipe/test_arrow_parquet.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
ogrinfo --format Arrow
if errorlevel 1 exit 1

ogrinfo --format Parquet
if errorlevel 1 exit 1
12 changes: 12 additions & 0 deletions recipe/test_arrow_parquet.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

# exit when any command fails
set -e
# print all commands
set -x

test -f ${PREFIX}/lib/gdalplugins/ogr_Arrow${SHLIB_EXT}
test -f ${PREFIX}/lib/gdalplugins/ogr_Parquet${SHLIB_EXT}

ogrinfo --format Arrow
ogrinfo --format Parquet

0 comments on commit f16f19f

Please sign in to comment.