ENH: Add test of Arrow roundtrip of boolean values, raise exception on GDAL <3.8.3 for bool values via Arrow API

.github/workflows/docker-gdal.yml

@@ -21,7 +21,7 @@ jobs:
       matrix:
         container:
           - "ghcr.io/osgeo/gdal:ubuntu-small-latest" # >= python 3.10.6
-          - "ghcr.io/osgeo/gdal:ubuntu-small-3.8.1" # python 3.10.12
+          - "ghcr.io/osgeo/gdal:ubuntu-small-3.8.3" # python 3.10.12
           - "ghcr.io/osgeo/gdal:ubuntu-small-3.7.3" # python 3.10.12
           - "ghcr.io/osgeo/gdal:ubuntu-small-3.6.4" # python 3.10.6
           - "osgeo/gdal:ubuntu-small-3.5.3" # python 3.8.10

CHANGES.md

@@ -6,6 +6,9 @@
 
 -   Fix error in `write_dataframe` if input has a date column and
     non-consecutive index values (#325).
+-   Raise exception in `read_arrow` or `read_dataframe(..., use_arrow=True)` if
+    a boolean column is detected due to error in GDAL reading boolean values (#335)
+    this has been fixed in GDAL >= 3.8.3.
 
 ## 0.7.2 (2023-10-30)
 

pyogrio/raw.py

@@ -258,6 +258,8 @@ def read_arrow(
     """
     from pyarrow import Table
 
+    gdal_version = get_gdal_version()
+
     if skip_features < 0:
         raise ValueError("'skip_features' must be >= 0")
 
@@ -275,7 +277,7 @@ def read_arrow(
 
     # handle skip_features internally within open_arrow if GDAL >= 3.8.0
     gdal_skip_features = 0
-    if get_gdal_version() >= (3, 8, 0):
+    if gdal_version >= (3, 8, 0):
         gdal_skip_features = skip_features
         skip_features = 0
 
@@ -328,6 +330,16 @@ def read_arrow(
         else:
             table = reader.read_all()
 
+    # raise error if schema has bool values and GDAL <3.8.3
+    # due to https://github.com/OSGeo/gdal/issues/8998
+    if gdal_version < (3, 8, 3) and any(
+        [col_type == "bool" for col_type in table.schema.types]
+    ):
+        raise RuntimeError(
+            "GDAL < 3.8.3 does not correctly read boolean data values using the "
+            "Arrow API.  Do not use read_arrow() / use_arrow=True for this dataset."
+        )
+
     return meta, table
 
 

pyogrio/tests/test_arrow.py

@@ -4,8 +4,9 @@
 
 import pytest
 
+import numpy as np
 from pyogrio import __gdal_version__, read_dataframe
-from pyogrio.raw import open_arrow, read_arrow
+from pyogrio.raw import open_arrow, read_arrow, write
 from pyogrio.tests.conftest import requires_arrow_api
 
 try:
@@ -205,3 +206,51 @@ def test_enable_with_environment_variable(test_ogr_types_list):
     with use_arrow_context():
         result = read_dataframe(test_ogr_types_list)
     assert "list_int64" in result.columns
+
+
+@pytest.mark.skipif(
+    __gdal_version__ < (3, 8, 3), reason="Arrow bool value bug fixed in GDAL >= 3.8.3"
+)
+def test_arrow_bool_roundtrip(tmpdir):
+    filename = os.path.join(str(tmpdir), "test.gpkg")
+
+    # Point(0, 0)
+    geometry = np.array(
+        [bytes.fromhex("010100000000000000000000000000000000000000")] * 5, dtype=object
+    )
+    bool_col = np.array([True, False, True, False, True])
+    field_data = [bool_col]
+    fields = ["bool_col"]
+
+    write(
+        filename, geometry, field_data, fields, geometry_type="Point", crs="EPSG:4326"
+    )
+    table = read_arrow(filename)[1]
+
+    assert np.array_equal(table["bool_col"].to_numpy(), bool_col)
+
+
+@pytest.mark.skipif(
+    __gdal_version__ >= (3, 8, 3), reason="Arrow bool value bug fixed in GDAL >= 3.8.3"
+)
+def test_arrow_bool_exception(tmpdir):
+    filename = os.path.join(str(tmpdir), "test.gpkg")
+
+    # Point(0, 0)
+    geometry = np.array(
+        [bytes.fromhex("010100000000000000000000000000000000000000")] * 5, dtype=object
+    )
+    bool_col = np.array([True, False, True, False, True])
+    field_data = [bool_col]
+    fields = ["bool_col"]
+
+    write(
+        filename, geometry, field_data, fields, geometry_type="Point", crs="EPSG:4326"
+    )
+
+    with pytest.raises(
+        RuntimeError,
+        match="GDAL < 3.8.3 does not correctly read boolean data values using "
+        "the Arrow API",
+    ):
+        read_arrow(filename)

pyogrio/tests/test_geopandas_io.py

@@ -1469,3 +1469,42 @@ def test_read_dataframe_arrow_dtypes(tmp_path):
     assert isinstance(result["col"].dtype, pd.ArrowDtype)
     result["col"] = result["col"].astype("float64")
     assert_geodataframe_equal(result, df)
+
+
+@requires_arrow_api
+@pytest.mark.skipif(
+    __gdal_version__ < (3, 8, 3), reason="Arrow bool value bug fixed in GDAL >= 3.8.3"
+)
+def test_arrow_bool_roundtrip(tmpdir):
+    filename = os.path.join(str(tmpdir), "test.gpkg")
+
+    df = gp.GeoDataFrame(
+        {"bool_col": [True, False, True, False, True], "geometry": [Point(0, 0)] * 5},
+        crs="EPSG:4326",
+    )
+
+    write_dataframe(df, filename)
+    result = read_dataframe(filename, use_arrow=True)
+    assert_geodataframe_equal(result, df)
+
+
+@requires_arrow_api
+@pytest.mark.skipif(
+    __gdal_version__ >= (3, 8, 3), reason="Arrow bool value bug fixed in GDAL >= 3.8.3"
+)
+def test_arrow_bool_exception(tmpdir):
+    filename = os.path.join(str(tmpdir), "test.gpkg")
+
+    df = gp.GeoDataFrame(
+        {"bool_col": [True, False, True, False, True], "geometry": [Point(0, 0)] * 5},
+        crs="EPSG:4326",
+    )
+
+    write_dataframe(df, filename)
+
+    with pytest.raises(
+        RuntimeError,
+        match="GDAL < 3.8.3 does not correctly read boolean data values using "
+        "the Arrow API",
+    ):
+        read_dataframe(filename, use_arrow=True)