Prepare 27.0.0 release (#423)

apache · Jul 3, 2023 · 3f81513 · 3f81513
1 parent c0be61b
commit 3f81513
Show file tree

Hide file tree

Showing 12 changed files with 337 additions and 243 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -17,7 +17,7 @@
 
 [package]
 name = "datafusion-python"
-version = "26.0.0"
+version = "27.0.0"
 homepage = "https://github.com/apache/arrow-datafusion-python"
 repository = "https://github.com/apache/arrow-datafusion-python"
 authors = ["Apache Arrow <dev@arrow.apache.org>"]
@@ -35,27 +35,27 @@ protoc = [ "datafusion-substrait/protoc" ]
 [dependencies]
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 rand = "0.8"
-pyo3 = { version = "0.18.1", features = ["extension-module", "abi3", "abi3-py37"] }
-datafusion = { version = "26.0.0" , features = ["pyarrow", "avro"] }
-datafusion-common = { version = "26.0.0", features = ["pyarrow"] }
-datafusion-expr = "26.0.0"
-datafusion-optimizer = "26.0.0"
-datafusion-sql = "26.0.0"
-datafusion-substrait = "26.0.0"
+pyo3 = { version = "0.19", features = ["extension-module", "abi3", "abi3-py38"] }
+datafusion = { version = "27.0.0" , features = ["pyarrow", "avro"] }
+datafusion-common = { version = "27.0.0", features = ["pyarrow"] }
+datafusion-expr = "27.0.0"
+datafusion-optimizer = "27.0.0"
+datafusion-sql = "27.0.0"
+datafusion-substrait = "27.0.0"
 prost = "0.11"
 prost-types = "0.11"
 uuid = { version = "1.3", features = ["v4"] }
 mimalloc = { version = "0.1", optional = true, default-features = false }
 async-trait = "0.1"
 futures = "0.3"
-object_store = { version = "0.5.4", features = ["aws", "gcp", "azure"] }
+object_store = { version = "0.6.1", features = ["aws", "gcp", "azure"] }
 parking_lot = "0.12"
 regex-syntax = "0.7.1"
 syn = "2.0.11"
 url = "2.2"
 
 [build-dependencies]
-pyo3-build-config = "0.18.3"
+pyo3-build-config = "0.19.0"
 
 [lib]
 name = "datafusion_python"

diff --git a/datafusion/tests/test_store.py b/datafusion/tests/test_store.py
@@ -37,7 +37,7 @@ def ctx(local):
 def test_read_parquet(ctx):
     ctx.register_parquet(
         "test",
-        f"file://{os.getcwd()}/testing/data/parquet",
+        f"file://{os.getcwd()}/parquet/data/alltypes_plain.parquet",
         [],
         True,
         ".parquet",

diff --git a/src/catalog.rs b/src/catalog.rs
@@ -24,8 +24,8 @@ use pyo3::prelude::*;
 use crate::errors::DataFusionError;
 use crate::utils::wait_for_future;
 use datafusion::{
-    arrow::pyarrow::PyArrowConvert,
-    catalog::{catalog::CatalogProvider, schema::SchemaProvider},
+    arrow::pyarrow::ToPyArrow,
+    catalog::{schema::SchemaProvider, CatalogProvider},
     datasource::{TableProvider, TableType},
 };
 

diff --git a/src/context.rs b/src/context.rs
@@ -40,8 +40,8 @@ use crate::utils::{get_tokio_runtime, wait_for_future};
 use datafusion::arrow::datatypes::{DataType, Schema};
 use datafusion::arrow::pyarrow::PyArrowType;
 use datafusion::arrow::record_batch::RecordBatch;
-use datafusion::datasource::datasource::TableProvider;
 use datafusion::datasource::MemTable;
+use datafusion::datasource::TableProvider;
 use datafusion::execution::context::{SessionConfig, SessionContext, TaskContext};
 use datafusion::execution::disk_manager::DiskManagerConfig;
 use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, UnboundedMemoryPool};

diff --git a/src/dataframe.rs b/src/dataframe.rs
@@ -20,7 +20,7 @@ use crate::sql::logical::PyLogicalPlan;
 use crate::utils::wait_for_future;
 use crate::{errors::DataFusionError, expr::PyExpr};
 use datafusion::arrow::datatypes::Schema;
-use datafusion::arrow::pyarrow::{PyArrowConvert, PyArrowType};
+use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow};
 use datafusion::arrow::util::pretty;
 use datafusion::dataframe::DataFrame;
 use datafusion::prelude::*;

diff --git a/src/dataset.rs b/src/dataset.rs
@@ -28,10 +28,10 @@ use async_trait::async_trait;
 
 use datafusion::arrow::datatypes::SchemaRef;
 use datafusion::arrow::pyarrow::PyArrowType;
-use datafusion::datasource::datasource::TableProviderFilterPushDown;
 use datafusion::datasource::{TableProvider, TableType};
 use datafusion::error::{DataFusionError, Result as DFResult};
 use datafusion::execution::context::SessionState;
+use datafusion::logical_expr::TableProviderFilterPushDown;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion_expr::Expr;
 

diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs
@@ -239,7 +239,7 @@ impl ExecutionPlan for DatasetExec {
         Python::with_gil(|py| {
             let number_of_fragments = self.fragments.as_ref(py).len();
             match t {
-                DisplayFormatType::Default => {
+                DisplayFormatType::Default | DisplayFormatType::Verbose => {
                     let projected_columns: Vec<String> = self
                         .schema
                         .fields()

diff --git a/src/physical_plan.rs b/src/physical_plan.rs
@@ -51,7 +51,7 @@ impl PyExecutionPlan {
 
     pub fn display_indent(&self) -> String {
         let d = displayable(self.plan.as_ref());
-        format!("{}", d.indent())
+        format!("{}", d.indent(false))
     }
 
     fn __repr__(&self) -> String {

diff --git a/src/record_batch.rs b/src/record_batch.rs
@@ -16,7 +16,7 @@
 // under the License.
 
 use crate::utils::wait_for_future;
-use datafusion::arrow::pyarrow::PyArrowConvert;
+use datafusion::arrow::pyarrow::ToPyArrow;
 use datafusion::arrow::record_batch::RecordBatch;
 use datafusion::physical_plan::SendableRecordBatchStream;
 use futures::StreamExt;

diff --git a/src/udaf.rs b/src/udaf.rs
@@ -21,10 +21,10 @@ use pyo3::{prelude::*, types::PyTuple};
 
 use datafusion::arrow::array::{Array, ArrayRef};
 use datafusion::arrow::datatypes::DataType;
-use datafusion::arrow::pyarrow::{PyArrowConvert, PyArrowType};
+use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow};
 use datafusion::common::ScalarValue;
 use datafusion::error::{DataFusionError, Result};
-use datafusion_expr::{create_udaf, Accumulator, AccumulatorFunctionImplementation, AggregateUDF};
+use datafusion_expr::{create_udaf, Accumulator, AccumulatorFactoryFunction, AggregateUDF};
 
 use crate::expr::PyExpr;
 use crate::utils::parse_volatility;
@@ -95,7 +95,7 @@ impl Accumulator for RustAccumulator {
     }
 }
 
-pub fn to_rust_accumulator(accum: PyObject) -> AccumulatorFunctionImplementation {
+pub fn to_rust_accumulator(accum: PyObject) -> AccumulatorFactoryFunction {
     Arc::new(move |_| -> Result<Box<dyn Accumulator>> {
         let accum = Python::with_gil(|py| {
             accum

diff --git a/src/udf.rs b/src/udf.rs
@@ -21,7 +21,7 @@ use pyo3::{prelude::*, types::PyTuple};
 
 use datafusion::arrow::array::{make_array, Array, ArrayData, ArrayRef};
 use datafusion::arrow::datatypes::DataType;
-use datafusion::arrow::pyarrow::{PyArrowConvert, PyArrowType};
+use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow};
 use datafusion::error::DataFusionError;
 use datafusion::physical_plan::functions::make_scalar_function;
 use datafusion::physical_plan::udf::ScalarUDF;