Merge pull request #138 from timvw/feat/bump-202501

Feat/bump 202501

Cargo.toml

@@ -25,15 +25,27 @@ aws-types = "1.2"
 aws-credential-types = "1.2"
 chrono = "0.4.38"
 clap = { version = "4.5.4", features = ["derive"] }
-datafusion = { version = "35", features = ["avro"] }
-deltalake = { version = "0.17", default-features = false, features = ["datafusion-ext", "s3", "gcs"] }
+#datafusion = { version = "44", features = ["avro"] }
+datafusion = { git = "https://github.com/timvw/datafusion.git", rev = "6e7c5467510636d97ee9ecf498cd54d7719f031d", features = ["avro"]}
+deltalake = { git = "https://github.com/delta-io/delta-rs.git", rev = "rust-v0.23.0", default-features = false, features = ["datafusion-ext", "s3", "gcs"] }
 futures = "0.3"
 glob = "0.3"
-object_store = { version = "0.9", features = ["aws", "gcp"] }
+object_store = { version = "0.11", features = ["aws", "gcp"] }
 regex = "1.10"
 tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
 url = "2.5"
 
+[patch.crates-io]
+datafusion = { git = "https://github.com/timvw/datafusion.git", rev = "6e7c5467510636d97ee9ecf498cd54d7719f031d", features = ["avro"]}
+datafusion-expr = { git = "https://github.com/timvw/datafusion.git", rev = "6e7c5467510636d97ee9ecf498cd54d7719f031d"}
+datafusion-common = { git = "https://github.com/timvw/datafusion.git", rev = "6e7c5467510636d97ee9ecf498cd54d7719f031d", features = ["avro"]}
+datafusion-proto = { git = "https://github.com/timvw/datafusion.git", rev = "6e7c5467510636d97ee9ecf498cd54d7719f031d"}
+datafusion-sql = { git = "https://github.com/timvw/datafusion.git", rev = "6e7c5467510636d97ee9ecf498cd54d7719f031d"}
+datafusion-physical-expr = { git = "https://github.com/timvw/datafusion.git", rev = "6e7c5467510636d97ee9ecf498cd54d7719f031d"}
+datafusion-physical-plan = { git = "https://github.com/timvw/datafusion.git", rev = "6e7c5467510636d97ee9ecf498cd54d7719f031d"}
+datafusion-functions = { git = "https://github.com/timvw/datafusion.git", rev = "6e7c5467510636d97ee9ecf498cd54d7719f031d"}
+datafusion-functions-aggregate = { git = "https://github.com/timvw/datafusion.git", rev = "6e7c5467510636d97ee9ecf498cd54d7719f031d"}
+
 [dev-dependencies]
 assert_cmd = "2.0.14"
 predicates = "3.1"

src/main.rs

@@ -8,7 +8,6 @@ use aws_sdk_glue::types::{StorageDescriptor, Table};
 use aws_sdk_glue::Client;
 use aws_types::SdkConfig;
 use clap::Parser;
-use datafusion::catalog::TableReference;
 use datafusion::common::{DataFusionError, Result};
 use datafusion::datasource::file_format::avro::AvroFormat;
 use datafusion::datasource::file_format::csv::CsvFormat;
@@ -20,6 +19,7 @@ use datafusion::datasource::listing::{
 };
 use datafusion::datasource::TableProvider;
 use datafusion::prelude::*;
+use datafusion::sql::TableReference;
 use deltalake::open_table;
 use object_store::aws::{AmazonS3, AmazonS3Builder};
 use object_store::gcp::{GoogleCloudStorage, GoogleCloudStorageBuilder};

tests/files_on_localfs.rs

@@ -136,7 +136,7 @@ async fn run_with_local_parquet_file() -> datafusion::common::Result<()> {
 async fn run_with_local_parquet_files_in_folder() -> datafusion::common::Result<()> {
     let mut cmd = get_qv_cmd()?;
     let cmd = cmd
-        .arg(&get_qv_testing_path("data/iceberg/db/COVID-19_NYT/data"))
+        .arg(get_qv_testing_path("data/iceberg/db/COVID-19_NYT/data"))
         .arg("-q")
         .arg("select * from tbl order by date, county, state, fips, cases, deaths");