Merge pull request #1709 from ion-elgreco/feat/enable_prebuffer_pyarrow

feat: improve read performance by 7x with prebuffer
delta-io · Oct 9, 2023 · ab6b0cf · ab6b0cf
2 parents 0a41ebc + 94b41b7
commit ab6b0cf
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/python/deltalake/table.py b/python/deltalake/table.py
@@ -24,6 +24,7 @@
     Expression,
     FileSystemDataset,
     ParquetFileFormat,
+    ParquetFragmentScanOptions,
     ParquetReadOptions,
 )
 
@@ -538,7 +539,10 @@ def to_pyarrow_dataset(
                 )
             )
 
-        format = ParquetFileFormat(read_options=parquet_read_options)
+        format = ParquetFileFormat(
+            read_options=parquet_read_options,
+            default_fragment_scan_options=ParquetFragmentScanOptions(pre_buffer=True),
+        )
 
         fragments = [
             format.make_fragment(

diff --git a/python/stubs/pyarrow/dataset.pyi b/python/stubs/pyarrow/dataset.pyi
@@ -6,6 +6,7 @@ Expression: Any
 field: Any
 partitioning: Any
 FileSystemDataset: Any
+ParquetFragmentScanOptions: Any
 ParquetFileFormat: Any
 ParquetReadOptions: Any
 ParquetFileWriteOptions: Any