Add param arrow_to_pandas_kwargs to read_dataframe + decrease memory usage

CHANGES.md

@@ -9,6 +9,8 @@
     unknown count for a data layer (e.g., OSM driver); this may have signficant
     performance impacts for some data sources that would otherwise return an
     unknown count (count is used in `read_info`, `read`, `read_dataframe`) (#271).
+-   Add `arrow_to_pandas_kwargs` parameter to `read_dataframe` + reduce memory usage
+    with `use_arrow=True` (#273)
 -   In `read_info`, the result now also contains the `total_bounds` of the layer as well
     as some extra `capabilities` of the data source driver (#281)
 -   Raise error if `read` or `read_dataframe` is called with parameters to read no

pyogrio/geopandas.py

@@ -44,6 +44,7 @@ def read_dataframe(
     sql_dialect=None,
     fid_as_index=False,
     use_arrow=False,
+    arrow_to_pandas_kwargs=None,
     **kwargs,
 ):
     """Read from an OGR data source to a GeoPandas GeoDataFrame or Pandas DataFrame.
@@ -130,10 +131,13 @@ def read_dataframe(
     fid_as_index : bool, optional (default: False)
         If True, will use the FIDs of the features that were read as the
         index of the GeoDataFrame.  May start at 0 or 1 depending on the driver.
-    use_arrow : bool, default False
+    use_arrow : bool, optional (default: False)
         Whether to use Arrow as the transfer mechanism of the read data
         from GDAL to Python (requires GDAL >= 3.6 and `pyarrow` to be
         installed). When enabled, this provides a further speed-up.
+    arrow_to_pandas_kwargs : dict, optional (default: None)
+        When use_arrow is True, these kwargs will be passed to the `to_pandas`_ ()
+        call for the arrow to pandas conversion.
     **kwargs
         Additional driver-specific dataset open options passed to OGR.  Invalid
         options will trigger a warning.
@@ -158,7 +162,11 @@ def read_dataframe(
 
         https://www.gaia-gis.it/gaia-sins/spatialite-sql-latest.html
 
-    """
+    .. _to_pandas:
+
+        https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.to_pandas
+
+    """  # noqa: E501
     if not HAS_GEOPANDAS:
         raise ImportError("geopandas is required to use pyogrio.read_dataframe()")
 
@@ -189,7 +197,15 @@ def read_dataframe(
 
     if use_arrow:
         meta, table = result
-        df = table.to_pandas()
+
+        # split_blocks and self_destruct decrease memory usage, but have as side effect
+        # that accessing table afterwards causes crash, so del table to avoid.
+        kwargs = {"self_destruct": True}
+        if arrow_to_pandas_kwargs is not None:
+            kwargs.update(arrow_to_pandas_kwargs)
+        df = table.to_pandas(**kwargs)
+        del table
+
         if fid_as_index:
             df = df.set_index(meta["fid_column"])
             df.index.names = ["fid"]

pyogrio/tests/test_arrow.py

@@ -63,6 +63,18 @@ def test_read_arrow_nested_types(test_ogr_types_list):
     assert result["list_int64"][0].tolist() == [0, 1]
 
 
+def test_read_arrow_to_pandas_kwargs(test_fgdb_vsi):
+    # with arrow, list types are supported
+    arrow_to_pandas_kwargs = {"strings_to_categorical": True}
+    result = read_dataframe(
+        test_fgdb_vsi,
+        use_arrow=True,
+        arrow_to_pandas_kwargs=arrow_to_pandas_kwargs,
+    )
+    assert "SEGMENT_NAME" in result.columns
+    assert result["SEGMENT_NAME"].dtype.name == "category"
+
+
 def test_read_arrow_raw(naturalearth_lowres):
     meta, table = read_arrow(naturalearth_lowres)
     assert isinstance(meta, dict)