test

delta-io · Dec 1, 2024 · ce74037 · ce74037
1 parent 7e5bdfe
commit ce74037
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 5 deletions.
diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs
@@ -542,6 +542,7 @@ impl<'a> DeltaScanBuilder<'a> {
             .filter
             .map(|expr| context.create_physical_expr(expr, &df_schema).unwrap());
 
+        // dbg!(logical_filter.clone());
         // Perform Pruning of files to scan
         let (files, files_scanned, files_pruned) = match self.files {
             Some(files) => {
@@ -550,9 +551,12 @@ impl<'a> DeltaScanBuilder<'a> {
                 (files, files_scanned, 0)
             }
             None => {
+                println!("im here now");
                 if let Some(predicate) = &logical_filter {
+                    // dbg!(logical_schema.clone());
                     let pruning_predicate =
                         PruningPredicate::try_new(predicate.clone(), logical_schema.clone())?;
+                    // dbg!(pruning_predicate.clone());
                     let files_to_prune = pruning_predicate.prune(self.snapshot)?;
                     let mut files_pruned = 0usize;
                     let files = self
@@ -570,6 +574,7 @@ impl<'a> DeltaScanBuilder<'a> {
                         .collect::<Vec<_>>();
 
                     let files_scanned = files.len();
+                    dbg!(files_scanned.clone());
                     (files, files_scanned, files_pruned)
                 } else {
                     let files = self.snapshot.file_actions()?;

diff --git a/crates/core/src/kernel/snapshot/log_data.rs b/crates/core/src/kernel/snapshot/log_data.rs
@@ -725,12 +725,14 @@ mod datafusion {
                 return None;
             }
             let expression = if self.metadata.partition_columns.contains(&column.name) {
-                Expression::column(vec![format!("add.partitionValues_parsed.{}", column.name)])
+                Expression::column(vec!["add", "partitionValues_parsed", &column.name])
             } else {
-                Expression::column(vec![format!(
-                    "add.stats_parsed.{}.{}",
-                    stats_field, column.name
-                )])
+                Expression::column(vec![
+                    "add",
+                    "stats_parsed",
+                    stats_field, 
+                    &column.name,
+                ])
             };
             let evaluator = ARROW_HANDLER.get_evaluator(
                 crate::kernel::models::fields::log_schema_ref().clone(),
@@ -751,6 +753,8 @@ mod datafusion {
                 results.push(result.record_batch().clone());
             }
             let batch = concat_batches(results[0].schema_ref(), &results).ok()?;
+
+            dbg!(batch.clone());
             batch.column_by_name("output").cloned()
         }
     }