Merge branch 'main' into test-machete

Author: samueleresca

Cargo.lock

@@ -60,7 +60,6 @@ apache-avro = { version = "0.20", default-features = false, features = [
 ], optional = true }
 arrow = { workspace = true }
 arrow-ipc = { workspace = true }
-base64 = "0.22.1"
 chrono = { workspace = true }
 half = { workspace = true }
 hashbrown = { workspace = true }

datafusion/common/Cargo.toml

@@ -17,7 +17,6 @@
 
 //! Options related to how parquet files should be written
 
-use base64::Engine;
 use std::sync::Arc;
 
 use crate::{
@@ -26,6 +25,7 @@ use crate::{
 };
 
 use arrow::datatypes::Schema;
+use parquet::arrow::encode_arrow_schema;
 // TODO: handle once deprecated
 #[allow(deprecated)]
 use parquet::{
@@ -166,31 +166,6 @@ impl TryFrom<&TableParquetOptions> for WriterPropertiesBuilder {
     }
 }
 
-/// Encodes the Arrow schema into the IPC format, and base64 encodes it
-///
-/// TODO: use extern parquet's private method, once publicly available.
-/// Refer to <https://github.com/apache/arrow-rs/pull/6916>
-fn encode_arrow_schema(schema: &Arc<Schema>) -> String {
-    let options = arrow_ipc::writer::IpcWriteOptions::default();
-    let mut dictionary_tracker = arrow_ipc::writer::DictionaryTracker::new(true);
-    let data_gen = arrow_ipc::writer::IpcDataGenerator::default();
-    let mut serialized_schema = data_gen.schema_to_bytes_with_dictionary_tracker(
-        schema,
-        &mut dictionary_tracker,
-        &options,
-    );
-
-    // manually prepending the length to the schema as arrow uses the legacy IPC format
-    // TODO: change after addressing ARROW-9777
-    let schema_len = serialized_schema.ipc_message.len();
-    let mut len_prefix_schema = Vec::with_capacity(schema_len + 8);
-    len_prefix_schema.append(&mut vec![255u8, 255, 255, 255]);
-    len_prefix_schema.append((schema_len as u32).to_le_bytes().to_vec().as_mut());
-    len_prefix_schema.append(&mut serialized_schema.ipc_message);
-
-    base64::prelude::BASE64_STANDARD.encode(&len_prefix_schema)
-}
-
 impl ParquetOptions {
     /// Convert the global session options, [`ParquetOptions`], into a single write action's [`WriterPropertiesBuilder`].
     ///

datafusion/common/src/file_options/parquet_writer.rs

@@ -344,3 +344,28 @@ async fn test_version_function() {
 
     assert_eq!(version.value(0), expected_version);
 }
+
+/// Regression test for https://github.com/apache/datafusion/issues/17513
+/// See https://github.com/apache/datafusion/pull/17520
+#[tokio::test]
+async fn test_select_no_projection() -> Result<()> {
+    let tmp_dir = TempDir::new()?;
+    // `create_ctx_with_partition` creates 10 rows per partition and we chose 1 partition
+    let ctx = create_ctx_with_partition(&tmp_dir, 1).await?;
+
+    let results = ctx.sql("SELECT FROM test").await?.collect().await?;
+    // We should get all of the rows, just without any columns
+    let total_rows: usize = results.iter().map(|b| b.num_rows()).sum();
+    assert_eq!(total_rows, 10);
+    // Check that none of the batches have any columns
+    for batch in &results {
+        assert_eq!(batch.num_columns(), 0);
+    }
+    // Sanity check the output, should be just empty columns
+    assert_snapshot!(batches_to_sort_string(&results), @r"
+    ++
+    ++
+    ++
+    ");
+    Ok(())
+}

datafusion/core/tests/sql/select.rs

@@ -676,14 +676,6 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
         let mut prepared_select_exprs = vec![];
         let mut error_builder = DataFusionErrorBuilder::new();
 
-        // Handle the case where no projection is specified but we have a valid FROM clause
-        // In this case, implicitly add a wildcard projection (SELECT *)
-        let projection = if projection.is_empty() && !empty_from {
-            vec![SelectItem::Wildcard(WildcardAdditionalOptions::default())]
-        } else {
-            projection
-        };
-
         for expr in projection {
             match self.sql_select_to_rex(expr, plan, empty_from, planner_context) {
                 Ok(expr) => prepared_select_exprs.push(expr),

datafusion/sql/src/select.rs

@@ -252,3 +252,31 @@ physical_plan
 
 statement ok
 drop table t;
+
+# Regression test for 
+# https://github.com/apache/datafusion/issues/17513
+
+query I
+COPY (select 1 as a, 2 as b)
+TO 'test_files/scratch/projection/17513.parquet'
+STORED AS PARQUET;
+----
+1
+
+statement ok
+create external table t1 stored as parquet location 'test_files/scratch/projection/17513.parquet';
+
+query TT
+explain format indent
+select from t1 where t1.a > 1;
+----
+logical_plan
+01)Projection:
+02)--Filter: t1.a > Int64(1)
+03)----TableScan: t1 projection=[a], partial_filters=[t1.a > Int64(1)]
+physical_plan
+01)ProjectionExec: expr=[]
+02)--CoalesceBatchesExec: target_batch_size=8192
+03)----FilterExec: a@0 > 1
+04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection/17513.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 > 1, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 > 1, required_guarantees=[]

datafusion/sqllogictest/test_files/from-first.slt

@@ -75,20 +75,6 @@ Example:
 SELECT t.a FROM table AS t
 ```
 
-The `FROM` clause can also come before the `SELECT` clause.
-Example:
-
-```sql
-FROM table AS t
-SELECT t.a
-```
-
-If the `SELECT` clause is omitted, the `FROM` clause will return all columns from the table.
-
-```sql
-FROM table
-```
-
 ## WHERE clause
 
 Example: