diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index e2cfa59f224f..2b58eca2f5eb 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -117,7 +117,7 @@ postgres-protocol = "0.6.4" postgres-types = { version = "0.2.4", features = ["derive", "with-chrono-0_4"] } rstest = "0.16.0" rust_decimal = { version = "1.27.0", features = ["tokio-pg"] } -sqllogictest = "0.11.1" +sqllogictest = "0.12.0" test-utils = { path = "../../test-utils" } thiserror = "1.0.37" tokio-postgres = "0.7.7" diff --git a/datafusion/core/tests/sqllogictests/src/engines/datafusion/create_table.rs b/datafusion/core/tests/sqllogictests/src/engines/datafusion/create_table.rs index 9b43b98cb802..b891ab8c8b6d 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/datafusion/create_table.rs +++ b/datafusion/core/tests/sqllogictests/src/engines/datafusion/create_table.rs @@ -16,8 +16,8 @@ // under the License. use super::error::Result; -use crate::engines::datafusion::error::DFSqlLogicTestError; use crate::engines::datafusion::util::LogicTestContextProvider; +use crate::{engines::datafusion::error::DFSqlLogicTestError, output::DFOutput}; use datafusion::datasource::MemTable; use datafusion::prelude::SessionContext; use datafusion_common::{DataFusionError, OwnedTableReference}; @@ -32,7 +32,7 @@ pub async fn create_table( columns: Vec, if_not_exists: bool, or_replace: bool, -) -> Result { +) -> Result { let table_reference = object_name_to_table_reference(name, ctx.enable_ident_normalization())?; let existing_table = ctx.table(&table_reference).await; @@ -60,7 +60,7 @@ fn create_new_table( ctx: &SessionContext, table_reference: OwnedTableReference, columns: Vec, -) -> Result { +) -> Result { let config = ctx.copied_config(); let sql_to_rel = SqlToRel::new_with_options( &LogicTestContextProvider {}, diff --git a/datafusion/core/tests/sqllogictests/src/engines/datafusion/insert.rs b/datafusion/core/tests/sqllogictests/src/engines/datafusion/insert.rs index 16fbfb7a5d40..cb8e15591d24 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/datafusion/insert.rs +++ b/datafusion/core/tests/sqllogictests/src/engines/datafusion/insert.rs @@ -16,7 +16,7 @@ // under the License. use super::error::Result; -use crate::engines::datafusion::util::LogicTestContextProvider; +use crate::{engines::datafusion::util::LogicTestContextProvider, output::DFOutput}; use arrow::record_batch::RecordBatch; use datafusion::datasource::MemTable; use datafusion::prelude::SessionContext; @@ -27,7 +27,7 @@ use sqllogictest::DBOutput; use sqlparser::ast::{Expr, SetExpr, Statement as SQLStatement}; use std::sync::Arc; -pub async fn insert(ctx: &SessionContext, insert_stmt: SQLStatement) -> Result { +pub async fn insert(ctx: &SessionContext, insert_stmt: SQLStatement) -> Result { // First, use sqlparser to get table name and insert values let table_reference; let insert_values: Vec>; diff --git a/datafusion/core/tests/sqllogictests/src/engines/datafusion/mod.rs b/datafusion/core/tests/sqllogictests/src/engines/datafusion/mod.rs index 7da453d775d9..918419370ffb 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/datafusion/mod.rs +++ b/datafusion/core/tests/sqllogictests/src/engines/datafusion/mod.rs @@ -18,7 +18,7 @@ use std::path::PathBuf; use std::time::Duration; -use sqllogictest::DBOutput; +use crate::output::{DFColumnType, DFOutput}; use self::error::{DFSqlLogicTestError, Result}; use async_trait::async_trait; @@ -49,8 +49,9 @@ impl DataFusion { #[async_trait] impl sqllogictest::AsyncDB for DataFusion { type Error = DFSqlLogicTestError; + type ColumnType = DFColumnType; - async fn run(&mut self, sql: &str) -> Result { + async fn run(&mut self, sql: &str) -> Result { println!( "[{}] Running query: \"{}\"", self.relative_path.display(), @@ -75,7 +76,7 @@ impl sqllogictest::AsyncDB for DataFusion { } } -async fn run_query(ctx: &SessionContext, sql: impl Into) -> Result { +async fn run_query(ctx: &SessionContext, sql: impl Into) -> Result { let sql = sql.into(); // Check if the sql is `insert` if let Ok(mut statements) = DFParser::parse_sql(&sql) { diff --git a/datafusion/core/tests/sqllogictests/src/engines/datafusion/normalize.rs b/datafusion/core/tests/sqllogictests/src/engines/datafusion/normalize.rs index 148b4ecf7b88..601033e6ac3c 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/datafusion/normalize.rs +++ b/datafusion/core/tests/sqllogictests/src/engines/datafusion/normalize.rs @@ -17,7 +17,9 @@ use arrow::{array, array::ArrayRef, datatypes::DataType, record_batch::RecordBatch}; use datafusion::error::DataFusionError; -use sqllogictest::{ColumnType, DBOutput}; +use sqllogictest::DBOutput; + +use crate::output::{DFColumnType, DFOutput}; use super::super::conversion::*; use super::error::{DFSqlLogicTestError, Result}; @@ -26,7 +28,7 @@ use super::error::{DFSqlLogicTestError, Result}; /// /// Assumes empty record batches are a successful statement completion /// -pub fn convert_batches(batches: Vec) -> Result { +pub fn convert_batches(batches: Vec) -> Result { if batches.is_empty() { // DataFusion doesn't report number of rows complete return Ok(DBOutput::StatementComplete(0)); @@ -36,7 +38,7 @@ pub fn convert_batches(batches: Vec) -> Result { // TODO: report the the actual types of the result // https://github.com/apache/arrow-datafusion/issues/4499 - let types = vec![ColumnType::Any; batches[0].num_columns()]; + let types = vec![DFColumnType::Any; batches[0].num_columns()]; let mut rows = vec![]; for batch in batches { diff --git a/datafusion/core/tests/sqllogictests/src/engines/postgres/mod.rs b/datafusion/core/tests/sqllogictests/src/engines/postgres/mod.rs index 84adde90d94d..5284111887fe 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/postgres/mod.rs +++ b/datafusion/core/tests/sqllogictests/src/engines/postgres/mod.rs @@ -22,9 +22,11 @@ use async_trait::async_trait; use bytes::Bytes; use futures::{SinkExt, StreamExt}; use log::debug; -use sqllogictest::{ColumnType, DBOutput}; +use sqllogictest::DBOutput; use tokio::task::JoinHandle; +use crate::output::{DFColumnType, DFOutput}; + use super::conversion::*; use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use postgres_types::Type; @@ -131,7 +133,7 @@ impl Postgres { /// ``` /// /// And read the file locally. - async fn run_copy_command(&mut self, sql: &str) -> Result { + async fn run_copy_command(&mut self, sql: &str) -> Result { let canonical_sql = sql.trim_start().to_ascii_lowercase(); debug!("Handling COPY command: {sql}"); @@ -256,8 +258,9 @@ fn cell_to_string(row: &Row, column: &Column, idx: usize) -> String { #[async_trait] impl sqllogictest::AsyncDB for Postgres { type Error = Error; + type ColumnType = DFColumnType; - async fn run(&mut self, sql: &str) -> Result { + async fn run(&mut self, sql: &str) -> Result { println!( "[{}] Running query: \"{}\"", self.relative_path.display(), @@ -301,12 +304,12 @@ impl sqllogictest::AsyncDB for Postgres { if output.is_empty() { let stmt = self.client.prepare(sql).await?; Ok(DBOutput::Rows { - types: vec![ColumnType::Any; stmt.columns().len()], + types: vec![DFColumnType::Any; stmt.columns().len()], rows: vec![], }) } else { Ok(DBOutput::Rows { - types: vec![ColumnType::Any; output[0].len()], + types: vec![DFColumnType::Any; output[0].len()], rows: output, }) } diff --git a/datafusion/core/tests/sqllogictests/src/main.rs b/datafusion/core/tests/sqllogictests/src/main.rs index ba79ff892b59..430d4b496660 100644 --- a/datafusion/core/tests/sqllogictests/src/main.rs +++ b/datafusion/core/tests/sqllogictests/src/main.rs @@ -26,6 +26,7 @@ use crate::engines::datafusion::DataFusion; use crate::engines::postgres::Postgres; mod engines; +mod output; mod setup; mod utils; @@ -90,7 +91,7 @@ async fn run_complete_file( path: &Path, relative_path: PathBuf, ) -> Result<(), Box> { - use sqllogictest::{default_validator, update_test_file}; + use sqllogictest::default_validator; info!("Using complete mode to complete: {}", path.display()); @@ -98,7 +99,8 @@ async fn run_complete_file( let mut runner = sqllogictest::Runner::new(DataFusion::new(ctx, relative_path)); let col_separator = " "; let validator = default_validator; - update_test_file(path, &mut runner, col_separator, validator) + runner + .update_test_file(path, col_separator, validator) .await .map_err(|e| e.to_string())?; diff --git a/datafusion/core/tests/sqllogictests/src/output.rs b/datafusion/core/tests/sqllogictests/src/output.rs new file mode 100644 index 000000000000..26d26eabaad1 --- /dev/null +++ b/datafusion/core/tests/sqllogictests/src/output.rs @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use sqllogictest::{DBOutput, DefaultColumnType}; + +pub type DFColumnType = DefaultColumnType; + +/// The output type fed to the sqllogictest Runner +/// +/// See for +/// potentially more full featured support. +pub type DFOutput = DBOutput;