From bb4d39d1cec84b7821b4b954b6f7b1bfd10c617f Mon Sep 17 00:00:00 2001 From: Edmondo Porcu Date: Wed, 15 May 2024 14:03:48 +0000 Subject: [PATCH 1/6] Example for simple conversion --- datafusion-examples/examples/plan_to_sql.rs | 44 +++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 datafusion-examples/examples/plan_to_sql.rs diff --git a/datafusion-examples/examples/plan_to_sql.rs b/datafusion-examples/examples/plan_to_sql.rs new file mode 100644 index 000000000000..84f66605871a --- /dev/null +++ b/datafusion-examples/examples/plan_to_sql.rs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion::error::Result; + +use datafusion::prelude::*; + +/// This example demonstrates the programmatic construction of +/// SQL using the DataFusion Expr [`Expr`] and LogicalPlan [`LogicalPlan`] API. +/// +/// +/// The code in this example shows how to: +/// 1. Create SQL from a variety of Expr and LogicalPlan: [`main`]` +/// 2. Create a simple expression [`Exprs`] with fluent API +/// and convert to sql against data: [`simple_expr_to_sql_demo`] + +#[tokio::main] +async fn main() -> Result<()> { + // See how to evaluate expressions + simple_expr_to_sql_demo(); + + Ok(()) +} + +/// DataFusion can convert expressions to SQL +fn simple_expr_to_sql_demo() { + let expr = col("a").lt(lit(5)).or(col("a").eq(lit(8))); + let sql = expr.to_string(); + assert_eq!(sql, "a < 5 OR a = 8"); +} From a6e2f701647f5df9700bbf1ed4a038d206733446 Mon Sep 17 00:00:00 2001 From: Edmondo Porcu Date: Thu, 16 May 2024 01:30:38 +0000 Subject: [PATCH 2/6] Fixing example --- datafusion-examples/examples/plan_to_sql.rs | 45 ++++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/datafusion-examples/examples/plan_to_sql.rs b/datafusion-examples/examples/plan_to_sql.rs index 84f66605871a..1b0274b3fd47 100644 --- a/datafusion-examples/examples/plan_to_sql.rs +++ b/datafusion-examples/examples/plan_to_sql.rs @@ -18,6 +18,9 @@ use datafusion::error::Result; use datafusion::prelude::*; +use datafusion::sql::unparser::expr_to_sql; +use datafusion_sql::unparser::dialect::CustomDialect; +use datafusion_sql::unparser::Unparser; /// This example demonstrates the programmatic construction of /// SQL using the DataFusion Expr [`Expr`] and LogicalPlan [`LogicalPlan`] API. @@ -26,19 +29,49 @@ use datafusion::prelude::*; /// The code in this example shows how to: /// 1. Create SQL from a variety of Expr and LogicalPlan: [`main`]` /// 2. Create a simple expression [`Exprs`] with fluent API -/// and convert to sql against data: [`simple_expr_to_sql_demo`] +/// and convert to sql: [`simple_expr_to_sql_demo`] +/// 3. Create a simple expression [`Exprs`] with fluent API +/// and convert to sql without escaping column names: [`simple_expr_to_sql_demo_no_escape`] #[tokio::main] async fn main() -> Result<()> { // See how to evaluate expressions - simple_expr_to_sql_demo(); + simple_expr_to_sql_demo()?; + simple_expr_to_sql_demo_no_escape()?; + simple_expr_to_sql_demo_escape_mysql_style()?; + Ok(()) +} + +/// DataFusion can convert expressions to SQL, using column name escaping +/// PostgreSQL style. +fn simple_expr_to_sql_demo() -> Result<()> { + let expr = col("a").lt(lit(5)).or(col("a").eq(lit(8))); + let ast = expr_to_sql(&expr)?; + let sql = format!("{}", ast); + assert_eq!(sql, r#"(("a" < 5) OR ("a" = 8))"#); + Ok(()) +} +/// DataFusion can convert expressions to SQL without escaping column names using +/// using a custom dialect and an explicit unparser +fn simple_expr_to_sql_demo_no_escape() -> Result<()> { + let expr = col("a").lt(lit(5)).or(col("a").eq(lit(8))); + let dialect = CustomDialect::new(None); + let unparser = Unparser::new(&dialect); + let ast = unparser.expr_to_sql(&expr)?; + let sql = format!("{}", ast); + assert_eq!(sql, r#"((a < 5) OR (a = 8))"#); Ok(()) } -/// DataFusion can convert expressions to SQL -fn simple_expr_to_sql_demo() { +/// DataFusion can convert expressions to SQL without escaping column names using +/// using a custom dialect and an explicit unparser +fn simple_expr_to_sql_demo_escape_mysql_style() -> Result<()> { let expr = col("a").lt(lit(5)).or(col("a").eq(lit(8))); - let sql = expr.to_string(); - assert_eq!(sql, "a < 5 OR a = 8"); + let dialect = CustomDialect::new(Some('`')); + let unparser = Unparser::new(&dialect); + let ast = unparser.expr_to_sql(&expr)?; + let sql = format!("{}", ast); + assert_eq!(sql, r#"((`a` < 5) OR (`a` = 8))"#); + Ok(()) } From ee48ccdf675bea85ca45e6416608e86b9445789f Mon Sep 17 00:00:00 2001 From: Edmondo Porcu Date: Thu, 16 May 2024 01:32:08 +0000 Subject: [PATCH 3/6] Updating README --- datafusion-examples/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md index 4b0e64ebdb7e..2c6c79f97012 100644 --- a/datafusion-examples/README.md +++ b/datafusion-examples/README.md @@ -62,6 +62,7 @@ cargo run --example csv_sql - [`memtable.rs`](examples/memtable.rs): Create an query data in memory using SQL and `RecordBatch`es - [`parquet_sql.rs`](examples/parquet_sql.rs): Build and run a query plan from a SQL statement against a local Parquet file - [`parquet_sql_multiple_files.rs`](examples/parquet_sql_multiple_files.rs): Build and run a query plan from a SQL statement against multiple local Parquet files +- [`plan_to_sql.rs`](examples/plan_to_sql.rs): Generate SQL from Datafusion `Expr` and `LogicalPlan` - [`pruning.rs`](examples/parquet_sql.rs): Use pruning to rule out files based on statistics - [`query-aws-s3.rs`](examples/external_dependency/query-aws-s3.rs): Configure `object_store` and run a query against files stored in AWS S3 - [`query-http-csv.rs`](examples/query-http-csv.rs): Configure `object_store` and run a query against files vi HTTP From 757005210a7d3888301b169ff2b9f8c48bfe4cf9 Mon Sep 17 00:00:00 2001 From: Edmondo Porcu Date: Thu, 16 May 2024 06:42:15 -0700 Subject: [PATCH 4/6] Update plan_to_sql.rs Update file header with more details about the examples --- datafusion-examples/examples/plan_to_sql.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datafusion-examples/examples/plan_to_sql.rs b/datafusion-examples/examples/plan_to_sql.rs index 1b0274b3fd47..04ba7461d2c1 100644 --- a/datafusion-examples/examples/plan_to_sql.rs +++ b/datafusion-examples/examples/plan_to_sql.rs @@ -32,6 +32,9 @@ use datafusion_sql::unparser::Unparser; /// and convert to sql: [`simple_expr_to_sql_demo`] /// 3. Create a simple expression [`Exprs`] with fluent API /// and convert to sql without escaping column names: [`simple_expr_to_sql_demo_no_escape`] +/// 4. Create a simple expression [`Exprs`] with fluent API +/// and convert to sql escaping column names a MySQL style: [`simple_expr_to_sql_demo_escape_mysql_style`] + #[tokio::main] async fn main() -> Result<()> { From c9761281916a3f49c91ffe3ade3611dd62c1b053 Mon Sep 17 00:00:00 2001 From: Edmondo Porcu Date: Thu, 16 May 2024 14:48:20 +0000 Subject: [PATCH 5/6] Fixing formatting --- datafusion-examples/examples/plan_to_sql.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/datafusion-examples/examples/plan_to_sql.rs b/datafusion-examples/examples/plan_to_sql.rs index 04ba7461d2c1..3915d3991f76 100644 --- a/datafusion-examples/examples/plan_to_sql.rs +++ b/datafusion-examples/examples/plan_to_sql.rs @@ -35,7 +35,6 @@ use datafusion_sql::unparser::Unparser; /// 4. Create a simple expression [`Exprs`] with fluent API /// and convert to sql escaping column names a MySQL style: [`simple_expr_to_sql_demo_escape_mysql_style`] - #[tokio::main] async fn main() -> Result<()> { // See how to evaluate expressions From 01a8267de1d28ca51b54ae2faf358d29dc3dbe59 Mon Sep 17 00:00:00 2001 From: Edmondo Porcu Date: Thu, 16 May 2024 12:47:57 -0700 Subject: [PATCH 6/6] Update README.md Fixing extra - --- datafusion-examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md index 4e7948b14d76..778950cbf926 100644 --- a/datafusion-examples/README.md +++ b/datafusion-examples/README.md @@ -63,7 +63,7 @@ cargo run --example csv_sql - [`parquet_sql.rs`](examples/parquet_sql.rs): Build and run a query plan from a SQL statement against a local Parquet file - [`parquet_sql_multiple_files.rs`](examples/parquet_sql_multiple_files.rs): Build and run a query plan from a SQL statement against multiple local Parquet files - ['parquet_exec_visitor.rs'](examples/parquet_exec_visitor.rs): Extract statistics by visiting an ExecutionPlan after execution -- - [`plan_to_sql.rs`](examples/plan_to_sql.rs): Generate SQL from Datafusion `Expr` and `LogicalPlan` +- [`plan_to_sql.rs`](examples/plan_to_sql.rs): Generate SQL from Datafusion `Expr` and `LogicalPlan` - [`pruning.rs`](examples/parquet_sql.rs): Use pruning to rule out files based on statistics - [`query-aws-s3.rs`](examples/external_dependency/query-aws-s3.rs): Configure `object_store` and run a query against files stored in AWS S3 - [`query-http-csv.rs`](examples/query-http-csv.rs): Configure `object_store` and run a query against files vi HTTP