From 633fbcf007953c219dc4f6d6d0a61a550608fb68 Mon Sep 17 00:00:00 2001 From: wu Date: Tue, 20 Jun 2023 08:41:31 +0000 Subject: [PATCH 01/17] feat: except in select --- e2e_test/batch/basic/query.slt.part | 10 +++++ .../tests/testdata/input/basic_query.yaml | 6 +++ .../tests/testdata/output/basic_query.yaml | 9 +++++ src/frontend/src/binder/select.rs | 40 ++++++++++++++++++- src/meta/src/manager/catalog/utils.rs | 1 + src/sqlparser/src/ast/query.rs | 3 ++ src/sqlparser/src/parser.rs | 39 +++++++++++++++++- src/sqlparser/tests/testdata/select.yaml | 2 + 8 files changed, 107 insertions(+), 3 deletions(-) diff --git a/e2e_test/batch/basic/query.slt.part b/e2e_test/batch/basic/query.slt.part index 18fdab91f4f24..50ac84a4ce1bd 100644 --- a/e2e_test/batch/basic/query.slt.part +++ b/e2e_test/batch/basic/query.slt.part @@ -30,6 +30,16 @@ select count(*) from t3; ---- 1 +query III +select * except v1 from t3; +---- +2 NULL + +query III +select * except t3.v1 from t3; +---- +2 NULL + statement error Division by zero select v1/0 from t3; diff --git a/src/frontend/planner_test/tests/testdata/input/basic_query.yaml b/src/frontend/planner_test/tests/testdata/input/basic_query.yaml index 6bafd10d22e93..098da4843d076 100644 --- a/src/frontend/planner_test/tests/testdata/input/basic_query.yaml +++ b/src/frontend/planner_test/tests/testdata/input/basic_query.yaml @@ -28,6 +28,12 @@ expected_outputs: - stream_plan - batch_plan +- sql: | + create table t (v1 int, v2 double precision); + select * except v1 from t; + expected_outputs: + - stream_plan + - batch_plan - name: test boolean expression common factor extraction sql: | create table t (v1 Boolean, v2 Boolean, v3 Boolean); diff --git a/src/frontend/planner_test/tests/testdata/output/basic_query.yaml b/src/frontend/planner_test/tests/testdata/output/basic_query.yaml index f50612c95bc67..40a6a9fdaf28b 100644 --- a/src/frontend/planner_test/tests/testdata/output/basic_query.yaml +++ b/src/frontend/planner_test/tests/testdata/output/basic_query.yaml @@ -42,6 +42,15 @@ StreamMaterialize { columns: [v1, t._row_id(hidden)], stream_key: [t._row_id], pk_columns: [t._row_id], pk_conflict: "NoCheck" } └─StreamFilter { predicate: (t.v1 < 1:Int32) } └─StreamTableScan { table: t, columns: [t.v1, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } +- sql: | + create table t (v1 int, v2 double precision); + select * except v1 from t; + batch_plan: | + BatchExchange { order: [], dist: Single } + └─BatchScan { table: t, columns: [t.v2], distribution: SomeShard } + stream_plan: | + StreamMaterialize { columns: [v2, t._row_id(hidden)], stream_key: [t._row_id], pk_columns: [t._row_id], pk_conflict: "NoCheck" } + └─StreamTableScan { table: t, columns: [t.v2, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } - name: test boolean expression common factor extraction sql: | create table t (v1 Boolean, v2 Boolean, v3 Boolean); diff --git a/src/frontend/src/binder/select.rs b/src/frontend/src/binder/select.rs index 896cc117ad9b6..e27e55e06788a 100644 --- a/src/frontend/src/binder/select.rs +++ b/src/frontend/src/binder/select.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; +use std::collections::HashSet; use std::fmt::Debug; use itertools::Itertools; @@ -226,13 +227,15 @@ impl Binder { ) -> Result<(Vec, Vec>)> { let mut select_list = vec![]; let mut aliases = vec![]; + let mut is_except = false; for item in select_items { match item { - SelectItem::UnnamedExpr(expr) => { + SelectItem::UnnamedExpr(expr) => { let alias = derive_alias(&expr); let bound = self.bind_expr(expr)?; select_list.push(bound); aliases.push(alias); + } SelectItem::ExprWithAlias { expr, alias } => { check_valid_column_name(&alias.real_value())?; @@ -285,7 +288,6 @@ impl Binder { })); select_list.extend(exprs); aliases.extend(names); - // TODO: we will need to be able to handle wildcard expressions bound to aliases // in the future. We'd then need a `NaturalGroupContext` // bound to each alias to correctly disambiguate column @@ -294,8 +296,42 @@ impl Binder { // We may need to refactor `NaturalGroupContext` to become span aware in that // case. } + SelectItem::Except(expr) => { + is_except = true; + let alias = derive_alias(&expr); + let bound = self.bind_expr(expr)?; + select_list.push(bound); + aliases.push(alias); + } } } + if is_except { + let mut reverse_select_list = vec![]; + let mut reverse_aliases = vec![]; + let mut indices: HashSet = HashSet::new(); + for expr in &select_list { + if let ExprImpl::InputRef(inner) = expr { + indices.insert(inner.index); + } else { + unreachable!(); + } + } + let indices: Vec = select_list.into_iter().map(|expr| { + if let ExprImpl::InputRef(inner) = expr { + (*inner).index + } else { + unreachable!() + } + }).collect(); + let (exprs, names) = + Self::iter_bound_columns(self.context.columns[..].iter().filter(|c| { + !c.is_hidden + && !indices.contains(&c.index) + })); + reverse_select_list.extend(exprs); + reverse_aliases.extend(names); + return Ok((reverse_select_list, reverse_aliases)); + } Ok((select_list, aliases)) } diff --git a/src/meta/src/manager/catalog/utils.rs b/src/meta/src/manager/catalog/utils.rs index 1a07805b875d6..bc167550c39c8 100644 --- a/src/meta/src/manager/catalog/utils.rs +++ b/src/meta/src/manager/catalog/utils.rs @@ -344,6 +344,7 @@ impl QueryRewriter<'_> { fn visit_select_item(&self, select_item: &mut SelectItem) { match select_item { SelectItem::UnnamedExpr(expr) + | SelectItem::Except(expr) | SelectItem::ExprQualifiedWildcard(expr, _) | SelectItem::ExprWithAlias { expr, .. } => self.visit_expr(expr), SelectItem::QualifiedWildcard(_) | SelectItem::Wildcard => {} diff --git a/src/sqlparser/src/ast/query.rs b/src/sqlparser/src/ast/query.rs index 82bab6096105a..affa61e24f398 100644 --- a/src/sqlparser/src/ast/query.rs +++ b/src/sqlparser/src/ast/query.rs @@ -311,6 +311,8 @@ pub enum SelectItem { QualifiedWildcard(ObjectName), /// An unqualified `*` Wildcard, + /// select * except [ ] + Except(Expr), } impl fmt::Display for SelectItem { @@ -328,6 +330,7 @@ impl fmt::Display for SelectItem { ), SelectItem::QualifiedWildcard(prefix) => write!(f, "{}.*", prefix), SelectItem::Wildcard => write!(f, "*"), + SelectItem::Except(expr) => write!(f, "EXCEPT {}", expr), } } } diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs index 9d937cfe5ab63..1c83eb247bec3 100644 --- a/src/sqlparser/src/parser.rs +++ b/src/sqlparser/src/parser.rs @@ -3529,7 +3529,15 @@ impl Parser { pub fn parse_select(&mut self) -> Result { let distinct = self.parse_all_or_distinct_on()?; - let projection = self.parse_comma_separated(Parser::parse_select_item)?; + let mut projection = self.parse_comma_separated(Parser::parse_select_item)?; + let index = self.index; + if self.parse_keyword(Keyword::EXCEPT) { + if projection.len() == 1 && let SelectItem::Wildcard = projection[0] { + projection = self.parse_comma_separated(Parser::parse_table_or_column)?; + } else { + self.index = index; + } + } // Note that for keywords to be properly handled here, they need to be // added to `RESERVED_FOR_COLUMN_ALIAS` / `RESERVED_FOR_TABLE_ALIAS`, @@ -4240,6 +4248,35 @@ impl Parser { } } + pub fn parse_table_or_column(&mut self) -> Result { + let index = self.index; + + match self.next_token().token { + Token::Word(w) => { + let mut id_parts = vec![w.to_ident()?]; + while self.consume_token(&Token::Period) { + let token = self.next_token(); + match token.token { + Token::Word(w) => id_parts.push(w.to_ident()?), + unexpected => { + self.index = index; + return self.expected( + "an identifier", + unexpected.with_location(token.location)) + } + } + } + return Ok(SelectItem::Except(Expr::CompoundIdentifier(id_parts))); + } + _ => (), + } + self.index = index; + return self.expected( + "an identifier", + self.peek_token() + ) + } + /// Parse a comma-delimited list of projections after SELECT pub fn parse_select_item(&mut self) -> Result { match self.parse_wildcard_or_expr()? { diff --git a/src/sqlparser/tests/testdata/select.yaml b/src/sqlparser/tests/testdata/select.yaml index c8aed6d1d8caf..c56f8b3126402 100644 --- a/src/sqlparser/tests/testdata/select.yaml +++ b/src/sqlparser/tests/testdata/select.yaml @@ -37,6 +37,8 @@ formatted_sql: SELECT id, fname, lname FROM customer WHERE salary <> 'Not Provided' AND salary <> '' - input: SELECT id FROM customer WHERE NOT salary = '' formatted_sql: SELECT id FROM customer WHERE NOT salary = '' +- input: SELECT * EXCEPT v1 FROM foo + formatted_sql: SELECT EXCEPT v1 FROM foo - input: SELECT * FROM t LIMIT 1 FETCH FIRST ROWS ONLY error_msg: 'sql parser error: Cannot specify both LIMIT and FETCH' - input: SELECT * FROM t FETCH FIRST ROWS WITH TIES From 959cbbe9fb3f8e5753a2366415882556804c14cc Mon Sep 17 00:00:00 2001 From: wu Date: Tue, 20 Jun 2023 08:56:48 +0000 Subject: [PATCH 02/17] fix: format --- src/frontend/src/binder/select.rs | 33 ++++++++++++++++--------------- src/sqlparser/src/parser.rs | 32 ++++++++++++------------------ 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/src/frontend/src/binder/select.rs b/src/frontend/src/binder/select.rs index e27e55e06788a..423316d9875f0 100644 --- a/src/frontend/src/binder/select.rs +++ b/src/frontend/src/binder/select.rs @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use itertools::Itertools; @@ -230,12 +229,11 @@ impl Binder { let mut is_except = false; for item in select_items { match item { - SelectItem::UnnamedExpr(expr) => { + SelectItem::UnnamedExpr(expr) => { let alias = derive_alias(&expr); let bound = self.bind_expr(expr)?; select_list.push(bound); aliases.push(alias); - } SelectItem::ExprWithAlias { expr, alias } => { check_valid_column_name(&alias.real_value())?; @@ -316,18 +314,21 @@ impl Binder { unreachable!(); } } - let indices: Vec = select_list.into_iter().map(|expr| { - if let ExprImpl::InputRef(inner) = expr { - (*inner).index - } else { - unreachable!() - } - }).collect(); - let (exprs, names) = - Self::iter_bound_columns(self.context.columns[..].iter().filter(|c| { - !c.is_hidden - && !indices.contains(&c.index) - })); + let indices: Vec = select_list + .into_iter() + .map(|expr| { + if let ExprImpl::InputRef(inner) = expr { + inner.index + } else { + unreachable!() + } + }) + .collect(); + let (exprs, names) = Self::iter_bound_columns( + self.context.columns[..] + .iter() + .filter(|c| !c.is_hidden && !indices.contains(&c.index)), + ); reverse_select_list.extend(exprs); reverse_aliases.extend(names); return Ok((reverse_select_list, reverse_aliases)); diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs index 1c83eb247bec3..b4db734fbdcf9 100644 --- a/src/sqlparser/src/parser.rs +++ b/src/sqlparser/src/parser.rs @@ -4251,30 +4251,24 @@ impl Parser { pub fn parse_table_or_column(&mut self) -> Result { let index = self.index; - match self.next_token().token { - Token::Word(w) => { - let mut id_parts = vec![w.to_ident()?]; - while self.consume_token(&Token::Period) { - let token = self.next_token(); - match token.token { - Token::Word(w) => id_parts.push(w.to_ident()?), - unexpected => { - self.index = index; - return self.expected( - "an identifier", - unexpected.with_location(token.location)) - } + if let Token::Word(w) = self.next_token().token { + let mut id_parts = vec![w.to_ident()?]; + while self.consume_token(&Token::Period) { + let token = self.next_token(); + match token.token { + Token::Word(w) => id_parts.push(w.to_ident()?), + unexpected => { + self.index = index; + return self + .expected("an identifier", unexpected.with_location(token.location)); } } - return Ok(SelectItem::Except(Expr::CompoundIdentifier(id_parts))); } - _ => (), + return Ok(SelectItem::Except(Expr::CompoundIdentifier(id_parts))); } + self.index = index; - return self.expected( - "an identifier", - self.peek_token() - ) + self.expected("an identifier", self.peek_token()) } /// Parse a comma-delimited list of projections after SELECT From e16d5293d7a282a607d8965079a796dd2831816a Mon Sep 17 00:00:00 2001 From: wu Date: Tue, 20 Jun 2023 09:06:16 +0000 Subject: [PATCH 03/17] clean code --- src/frontend/src/binder/select.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/frontend/src/binder/select.rs b/src/frontend/src/binder/select.rs index 423316d9875f0..b281a101beab2 100644 --- a/src/frontend/src/binder/select.rs +++ b/src/frontend/src/binder/select.rs @@ -314,16 +314,6 @@ impl Binder { unreachable!(); } } - let indices: Vec = select_list - .into_iter() - .map(|expr| { - if let ExprImpl::InputRef(inner) = expr { - inner.index - } else { - unreachable!() - } - }) - .collect(); let (exprs, names) = Self::iter_bound_columns( self.context.columns[..] .iter() From 1777a888e87cbddd24782e95935cdd10d3924b47 Mon Sep 17 00:00:00 2001 From: wu Date: Tue, 20 Jun 2023 09:08:25 +0000 Subject: [PATCH 04/17] add comments --- src/sqlparser/src/parser.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs index b4db734fbdcf9..22610fe736da5 100644 --- a/src/sqlparser/src/parser.rs +++ b/src/sqlparser/src/parser.rs @@ -3530,6 +3530,8 @@ impl Parser { let distinct = self.parse_all_or_distinct_on()?; let mut projection = self.parse_comma_separated(Parser::parse_select_item)?; + + // only deals with 'select * except' here let index = self.index; if self.parse_keyword(Keyword::EXCEPT) { if projection.len() == 1 && let SelectItem::Wildcard = projection[0] { From 82f23c2bd3c0bd4c224e60f0d44d39459b74cc41 Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 21 Jun 2023 02:53:01 +0000 Subject: [PATCH 05/17] fix: fix grammar --- src/frontend/src/binder/expr/function.rs | 1 + src/frontend/src/binder/select.rs | 44 ++++++++------------- src/meta/src/manager/catalog/utils.rs | 11 +++++- src/sqlparser/src/ast/mod.rs | 13 +++++++ src/sqlparser/src/ast/query.rs | 17 ++++++-- src/sqlparser/src/parser.rs | 49 +++++++----------------- 6 files changed, 69 insertions(+), 66 deletions(-) diff --git a/src/frontend/src/binder/expr/function.rs b/src/frontend/src/binder/expr/function.rs index e4db03386a6bf..e0a864b86b8cb 100644 --- a/src/frontend/src/binder/expr/function.rs +++ b/src/frontend/src/binder/expr/function.rs @@ -1011,6 +1011,7 @@ impl Binder { FunctionArgExpr::QualifiedWildcard(_) => todo!(), FunctionArgExpr::ExprQualifiedWildcard(_, _) => todo!(), FunctionArgExpr::Wildcard => Ok(vec![]), + FunctionArgExpr::Except(_) => unreachable!(), } } diff --git a/src/frontend/src/binder/select.rs b/src/frontend/src/binder/select.rs index b281a101beab2..f599fa0441a30 100644 --- a/src/frontend/src/binder/select.rs +++ b/src/frontend/src/binder/select.rs @@ -226,7 +226,6 @@ impl Binder { ) -> Result<(Vec, Vec>)> { let mut select_list = vec![]; let mut aliases = vec![]; - let mut is_except = false; for item in select_items { match item { SelectItem::UnnamedExpr(expr) => { @@ -294,34 +293,25 @@ impl Binder { // We may need to refactor `NaturalGroupContext` to become span aware in that // case. } - SelectItem::Except(expr) => { - is_except = true; - let alias = derive_alias(&expr); - let bound = self.bind_expr(expr)?; - select_list.push(bound); - aliases.push(alias); - } - } - } - if is_except { - let mut reverse_select_list = vec![]; - let mut reverse_aliases = vec![]; - let mut indices: HashSet = HashSet::new(); - for expr in &select_list { - if let ExprImpl::InputRef(inner) = expr { - indices.insert(inner.index); - } else { - unreachable!(); + SelectItem::Except(exprs) => { + let mut indices = HashSet::new(); + for expr in exprs { + let bound = self.bind_expr(expr)?; + if let ExprImpl::InputRef(inner) = bound { + indices.insert(inner.index); + } else { + unreachable!(); + } + } + let (exprs, names) = Self::iter_bound_columns( + self.context.columns[..] + .iter() + .filter(|c| !c.is_hidden && !indices.contains(&c.index)), + ); + select_list.extend(exprs); + aliases.extend(names); } } - let (exprs, names) = Self::iter_bound_columns( - self.context.columns[..] - .iter() - .filter(|c| !c.is_hidden && !indices.contains(&c.index)), - ); - reverse_select_list.extend(exprs); - reverse_aliases.extend(names); - return Ok((reverse_select_list, reverse_aliases)); } Ok((select_list, aliases)) } diff --git a/src/meta/src/manager/catalog/utils.rs b/src/meta/src/manager/catalog/utils.rs index bc167550c39c8..03ba293f4ec9f 100644 --- a/src/meta/src/manager/catalog/utils.rs +++ b/src/meta/src/manager/catalog/utils.rs @@ -245,6 +245,11 @@ impl QueryRewriter<'_> { self.visit_expr(expr) } FunctionArgExpr::QualifiedWildcard(_) | FunctionArgExpr::Wildcard => {} + FunctionArgExpr::Except(exprs) => { + for expr in exprs { + self.visit_expr(expr); + } + } }, } } @@ -344,10 +349,14 @@ impl QueryRewriter<'_> { fn visit_select_item(&self, select_item: &mut SelectItem) { match select_item { SelectItem::UnnamedExpr(expr) - | SelectItem::Except(expr) | SelectItem::ExprQualifiedWildcard(expr, _) | SelectItem::ExprWithAlias { expr, .. } => self.visit_expr(expr), SelectItem::QualifiedWildcard(_) | SelectItem::Wildcard => {} + SelectItem::Except(exprs) => { + for expr in exprs { + self.visit_expr(expr); + } + } } } } diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs index 0ad6591eb9a34..664751ea53c2e 100644 --- a/src/sqlparser/src/ast/mod.rs +++ b/src/sqlparser/src/ast/mod.rs @@ -1919,6 +1919,7 @@ pub enum FunctionArgExpr { QualifiedWildcard(ObjectName), /// An unqualified `*` Wildcard, + Except(Vec), } impl fmt::Display for FunctionArgExpr { @@ -1937,6 +1938,18 @@ impl fmt::Display for FunctionArgExpr { } FunctionArgExpr::QualifiedWildcard(prefix) => write!(f, "{}.*", prefix), FunctionArgExpr::Wildcard => f.write_str("*"), + FunctionArgExpr::Except(exprs) => { + write!( + f, + "EXCEPT {}", + exprs + .iter() + .map(|v| v.to_string()) + .collect::>() + .as_slice() + .join(", ") + ) + } } } } diff --git a/src/sqlparser/src/ast/query.rs b/src/sqlparser/src/ast/query.rs index affa61e24f398..f482458110fe8 100644 --- a/src/sqlparser/src/ast/query.rs +++ b/src/sqlparser/src/ast/query.rs @@ -311,8 +311,8 @@ pub enum SelectItem { QualifiedWildcard(ObjectName), /// An unqualified `*` Wildcard, - /// select * except [ ] - Except(Expr), + /// select * except ( test ) + Except(Vec) } impl fmt::Display for SelectItem { @@ -330,7 +330,18 @@ impl fmt::Display for SelectItem { ), SelectItem::QualifiedWildcard(prefix) => write!(f, "{}.*", prefix), SelectItem::Wildcard => write!(f, "*"), - SelectItem::Except(expr) => write!(f, "EXCEPT {}", expr), + SelectItem::Except(exprs) => { + write!( + f, + "* EXCEPT {}", + exprs + .iter() + .map(|v| v.to_string()) + .collect::>() + .as_slice() + .join(", ") + ) + } } } } diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs index 22610fe736da5..99c46d8df67e2 100644 --- a/src/sqlparser/src/parser.rs +++ b/src/sqlparser/src/parser.rs @@ -83,6 +83,7 @@ pub enum WildcardOrExpr { ExprQualifiedWildcard(Expr, Vec), QualifiedWildcard(ObjectName), Wildcard, + Except(Vec), } impl From for FunctionArgExpr { @@ -94,6 +95,7 @@ impl From for FunctionArgExpr { } WildcardOrExpr::QualifiedWildcard(prefix) => Self::QualifiedWildcard(prefix), WildcardOrExpr::Wildcard => Self::Wildcard, + WildcardOrExpr::Except(exprs) => Self::Except(exprs), } } } @@ -313,7 +315,14 @@ impl Parser { return self.word_concat_wildcard_expr(w.to_ident()?, wildcard_expr); } Token::Mul => { - return Ok(WildcardOrExpr::Wildcard); + if self.parse_keyword(Keyword::EXCEPT) && self.consume_token(&Token::LParen) { + let exprs = self.parse_comma_separated(Parser::parse_expr)?; + if self.consume_token(&Token::RParen) { + return Ok(WildcardOrExpr::Except(exprs)); + } + } else { + return Ok(WildcardOrExpr::Wildcard); + } } // parses wildcard field selection expression. // Code is similar to `parse_struct_selection` @@ -349,6 +358,7 @@ impl Parser { WildcardOrExpr::Wildcard => {} WildcardOrExpr::ExprQualifiedWildcard(_, _) => unreachable!(), WildcardOrExpr::Expr(e) => return Ok(WildcardOrExpr::Expr(e)), + WildcardOrExpr::Except(_) => unreachable!(), } Ok(WildcardOrExpr::QualifiedWildcard(ObjectName(idents))) } @@ -390,6 +400,7 @@ impl Parser { WildcardOrExpr::Wildcard => {} WildcardOrExpr::ExprQualifiedWildcard(_, _) => unreachable!(), WildcardOrExpr::Expr(_) => unreachable!(), + WildcardOrExpr::Except(_) => unreachable!(), } Ok(WildcardOrExpr::ExprQualifiedWildcard(expr, idents)) } @@ -3529,17 +3540,7 @@ impl Parser { pub fn parse_select(&mut self) -> Result { let distinct = self.parse_all_or_distinct_on()?; - let mut projection = self.parse_comma_separated(Parser::parse_select_item)?; - - // only deals with 'select * except' here - let index = self.index; - if self.parse_keyword(Keyword::EXCEPT) { - if projection.len() == 1 && let SelectItem::Wildcard = projection[0] { - projection = self.parse_comma_separated(Parser::parse_table_or_column)?; - } else { - self.index = index; - } - } + let projection = self.parse_comma_separated(Parser::parse_select_item)?; // Note that for keywords to be properly handled here, they need to be // added to `RESERVED_FOR_COLUMN_ALIAS` / `RESERVED_FOR_TABLE_ALIAS`, @@ -4250,29 +4251,6 @@ impl Parser { } } - pub fn parse_table_or_column(&mut self) -> Result { - let index = self.index; - - if let Token::Word(w) = self.next_token().token { - let mut id_parts = vec![w.to_ident()?]; - while self.consume_token(&Token::Period) { - let token = self.next_token(); - match token.token { - Token::Word(w) => id_parts.push(w.to_ident()?), - unexpected => { - self.index = index; - return self - .expected("an identifier", unexpected.with_location(token.location)); - } - } - } - return Ok(SelectItem::Except(Expr::CompoundIdentifier(id_parts))); - } - - self.index = index; - self.expected("an identifier", self.peek_token()) - } - /// Parse a comma-delimited list of projections after SELECT pub fn parse_select_item(&mut self) -> Result { match self.parse_wildcard_or_expr()? { @@ -4287,6 +4265,7 @@ impl Parser { Ok(SelectItem::ExprQualifiedWildcard(expr, prefix)) } WildcardOrExpr::Wildcard => Ok(SelectItem::Wildcard), + WildcardOrExpr::Except(exprs) => Ok(SelectItem::Except(exprs)), } } From c674d5b56eb744f5b6dba5335294997561715070 Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 21 Jun 2023 02:53:38 +0000 Subject: [PATCH 06/17] add test --- e2e_test/batch/basic/query.slt.part | 9 +++++++-- e2e_test/batch/basic/subquery.slt.part | 17 +++++++++++++++++ e2e_test/batch/top_n/group_top_n.slt | 13 +++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/e2e_test/batch/basic/query.slt.part b/e2e_test/batch/basic/query.slt.part index 50ac84a4ce1bd..7f0eb1f6f755f 100644 --- a/e2e_test/batch/basic/query.slt.part +++ b/e2e_test/batch/basic/query.slt.part @@ -31,15 +31,20 @@ select count(*) from t3; 1 query III -select * except v1 from t3; +select * except (v1) from t3; ---- 2 NULL query III -select * except t3.v1 from t3; +select * except (t3.v1) from t3; ---- 2 NULL +query III +select * except (v1), * except (v2) from t3; +---- +2 NULL 1 NULL + statement error Division by zero select v1/0 from t3; diff --git a/e2e_test/batch/basic/subquery.slt.part b/e2e_test/batch/basic/subquery.slt.part index b5f8d1f7fa94e..a278b1a2eee99 100644 --- a/e2e_test/batch/basic/subquery.slt.part +++ b/e2e_test/batch/basic/subquery.slt.part @@ -71,6 +71,23 @@ NULL 1 NULL 2 NULL NULL +query II +select * except (b,d) from (select t1.x as a, t1.y as b, t2.x as c, t2.y as d from t1 join t2 on t1.x = t2.x where t1.x=1); +---- +1 1 +1 1 +1 1 +1 1 + +query II +select * except (t1.x, t2.y), * except (t1.y, t2.x) from t1 join t2 on t1.y = t2.y where exists(select * from t3 where t1.x = t3.x and t2.y = t3.y) order by t2.x; +---- +2 1 2 2 +2 2 2 2 +2 NULL 2 2 + +query II +select * except (t1) statement ok drop table t1; diff --git a/e2e_test/batch/top_n/group_top_n.slt b/e2e_test/batch/top_n/group_top_n.slt index 628e03d5b97fb..71904968200cf 100644 --- a/e2e_test/batch/top_n/group_top_n.slt +++ b/e2e_test/batch/top_n/group_top_n.slt @@ -54,6 +54,19 @@ where rank <= 3 AND rank > 1; 3 2 3 3 +query II rowsort +select * except (rank) from ( + select *, ROW_NUMBER() OVER (PARTITION BY x ORDER BY y) as rank from t +) +where rank <= 3 AND rank > 1; +---- +1 2 +1 3 +2 2 +2 3 +3 2 +3 3 + query II rowsort select x, y from ( select *, RANK() OVER (ORDER BY y) as rank from t From 2d42799beba0be789ea6ff0b935f955979c3f3e7 Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 21 Jun 2023 03:05:24 +0000 Subject: [PATCH 07/17] fix format --- .../planner_test/tests/testdata/input/basic_query.yaml | 4 ++-- .../tests/testdata/output/basic_query.yaml | 10 +++++----- src/frontend/src/binder/select.rs | 4 ++-- src/sqlparser/src/ast/query.rs | 4 ++-- src/sqlparser/tests/testdata/select.yaml | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/frontend/planner_test/tests/testdata/input/basic_query.yaml b/src/frontend/planner_test/tests/testdata/input/basic_query.yaml index 098da4843d076..87370a3100a29 100644 --- a/src/frontend/planner_test/tests/testdata/input/basic_query.yaml +++ b/src/frontend/planner_test/tests/testdata/input/basic_query.yaml @@ -29,8 +29,8 @@ - stream_plan - batch_plan - sql: | - create table t (v1 int, v2 double precision); - select * except v1 from t; + create table t (v1 int, v2 int, v3 int); + select * except (v1, v2) from t; expected_outputs: - stream_plan - batch_plan diff --git a/src/frontend/planner_test/tests/testdata/output/basic_query.yaml b/src/frontend/planner_test/tests/testdata/output/basic_query.yaml index 40a6a9fdaf28b..f271321cc0ce0 100644 --- a/src/frontend/planner_test/tests/testdata/output/basic_query.yaml +++ b/src/frontend/planner_test/tests/testdata/output/basic_query.yaml @@ -43,14 +43,14 @@ └─StreamFilter { predicate: (t.v1 < 1:Int32) } └─StreamTableScan { table: t, columns: [t.v1, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } - sql: | - create table t (v1 int, v2 double precision); - select * except v1 from t; + create table t (v1 int, v2 int, v3 int); + select * except (v1, v2) from t; batch_plan: | BatchExchange { order: [], dist: Single } - └─BatchScan { table: t, columns: [t.v2], distribution: SomeShard } + └─BatchScan { table: t, columns: [t.v3], distribution: SomeShard } stream_plan: | - StreamMaterialize { columns: [v2, t._row_id(hidden)], stream_key: [t._row_id], pk_columns: [t._row_id], pk_conflict: "NoCheck" } - └─StreamTableScan { table: t, columns: [t.v2, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } + StreamMaterialize { columns: [v3, t._row_id(hidden)], stream_key: [t._row_id], pk_columns: [t._row_id], pk_conflict: "NoCheck" } + └─StreamTableScan { table: t, columns: [t.v3, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } - name: test boolean expression common factor extraction sql: | create table t (v1 Boolean, v2 Boolean, v3 Boolean); diff --git a/src/frontend/src/binder/select.rs b/src/frontend/src/binder/select.rs index f599fa0441a30..80c3f3aec3dd8 100644 --- a/src/frontend/src/binder/select.rs +++ b/src/frontend/src/binder/select.rs @@ -301,13 +301,13 @@ impl Binder { indices.insert(inner.index); } else { unreachable!(); - } + } } let (exprs, names) = Self::iter_bound_columns( self.context.columns[..] .iter() .filter(|c| !c.is_hidden && !indices.contains(&c.index)), - ); + ); select_list.extend(exprs); aliases.extend(names); } diff --git a/src/sqlparser/src/ast/query.rs b/src/sqlparser/src/ast/query.rs index f482458110fe8..b3dfd114f126b 100644 --- a/src/sqlparser/src/ast/query.rs +++ b/src/sqlparser/src/ast/query.rs @@ -312,7 +312,7 @@ pub enum SelectItem { /// An unqualified `*` Wildcard, /// select * except ( test ) - Except(Vec) + Except(Vec), } impl fmt::Display for SelectItem { @@ -333,7 +333,7 @@ impl fmt::Display for SelectItem { SelectItem::Except(exprs) => { write!( f, - "* EXCEPT {}", + "* EXCEPT ({})", exprs .iter() .map(|v| v.to_string()) diff --git a/src/sqlparser/tests/testdata/select.yaml b/src/sqlparser/tests/testdata/select.yaml index c56f8b3126402..044d48f7d857c 100644 --- a/src/sqlparser/tests/testdata/select.yaml +++ b/src/sqlparser/tests/testdata/select.yaml @@ -37,8 +37,8 @@ formatted_sql: SELECT id, fname, lname FROM customer WHERE salary <> 'Not Provided' AND salary <> '' - input: SELECT id FROM customer WHERE NOT salary = '' formatted_sql: SELECT id FROM customer WHERE NOT salary = '' -- input: SELECT * EXCEPT v1 FROM foo - formatted_sql: SELECT EXCEPT v1 FROM foo +- input: SELECT * EXCEPT (v1,v2) FROM foo + formatted_sql: SELECT * EXCEPT (v1, v2) FROM foo - input: SELECT * FROM t LIMIT 1 FETCH FIRST ROWS ONLY error_msg: 'sql parser error: Cannot specify both LIMIT and FETCH' - input: SELECT * FROM t FETCH FIRST ROWS WITH TIES From 3c3115b5ce95788f67d1e49839ade5f71077313c Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 21 Jun 2023 03:06:40 +0000 Subject: [PATCH 08/17] fix test --- e2e_test/batch/basic/subquery.slt.part | 3 --- 1 file changed, 3 deletions(-) diff --git a/e2e_test/batch/basic/subquery.slt.part b/e2e_test/batch/basic/subquery.slt.part index a278b1a2eee99..dc27649a90b6d 100644 --- a/e2e_test/batch/basic/subquery.slt.part +++ b/e2e_test/batch/basic/subquery.slt.part @@ -86,9 +86,6 @@ select * except (t1.x, t2.y), * except (t1.y, t2.x) from t1 join t2 on t1.y = t2 2 2 2 2 2 NULL 2 2 -query II -select * except (t1) - statement ok drop table t1; From 81159cde21797dd57c4815bf2d3f47c227be8001 Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 21 Jun 2023 03:09:14 +0000 Subject: [PATCH 09/17] fix format --- src/sqlparser/src/ast/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs index 664751ea53c2e..ec3c1df1f93c6 100644 --- a/src/sqlparser/src/ast/mod.rs +++ b/src/sqlparser/src/ast/mod.rs @@ -1941,7 +1941,7 @@ impl fmt::Display for FunctionArgExpr { FunctionArgExpr::Except(exprs) => { write!( f, - "EXCEPT {}", + "EXCEPT ({})", exprs .iter() .map(|v| v.to_string()) From 40e7f3519614b80ff068aeadcbbb1c01fb703b6f Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 21 Jun 2023 08:45:34 +0000 Subject: [PATCH 10/17] combine except and wildcard --- src/frontend/src/binder/expr/function.rs | 4 +- src/frontend/src/binder/select.rs | 104 ++++++++++++----------- src/frontend/src/handler/create_sink.rs | 2 +- src/meta/src/manager/catalog/utils.rs | 8 +- src/sqlparser/src/ast/mod.rs | 15 ++-- src/sqlparser/src/ast/query.rs | 32 +++---- src/sqlparser/src/parser.rs | 24 +++--- src/tests/sqlsmith/src/sql_gen/query.rs | 46 +++++++++- 8 files changed, 139 insertions(+), 96 deletions(-) diff --git a/src/frontend/src/binder/expr/function.rs b/src/frontend/src/binder/expr/function.rs index e0a864b86b8cb..24b4ff71a0d36 100644 --- a/src/frontend/src/binder/expr/function.rs +++ b/src/frontend/src/binder/expr/function.rs @@ -1010,8 +1010,8 @@ impl Binder { FunctionArgExpr::Expr(expr) => Ok(vec![self.bind_expr_inner(expr)?]), FunctionArgExpr::QualifiedWildcard(_) => todo!(), FunctionArgExpr::ExprQualifiedWildcard(_, _) => todo!(), - FunctionArgExpr::Wildcard => Ok(vec![]), - FunctionArgExpr::Except(_) => unreachable!(), + FunctionArgExpr::WildcardOrWithExcept(None) => Ok(vec![]), + FunctionArgExpr::WildcardOrWithExcept(Some(_)) => unreachable!(), } } diff --git a/src/frontend/src/binder/select.rs b/src/frontend/src/binder/select.rs index 80c3f3aec3dd8..ed26ff1776fc6 100644 --- a/src/frontend/src/binder/select.rs +++ b/src/frontend/src/binder/select.rs @@ -259,57 +259,63 @@ impl Binder { select_list.extend(exprs); aliases.extend(names); } - SelectItem::Wildcard => { - if self.context.range_of.is_empty() { - return Err(ErrorCode::BindError( - "SELECT * with no tables specified is not valid".into(), - ) - .into()); - } - // Bind the column groups - // In psql, the USING and NATURAL columns come before the rest of the columns in - // a SELECT * statement - let (exprs, names) = self.iter_column_groups(); - select_list.extend(exprs); - aliases.extend(names); - - // Bind columns that are not in groups - let (exprs, names) = - Self::iter_bound_columns(self.context.columns[..].iter().filter(|c| { - !c.is_hidden - && !self - .context - .column_group_context - .mapping - .contains_key(&c.index) - })); - select_list.extend(exprs); - aliases.extend(names); - // TODO: we will need to be able to handle wildcard expressions bound to aliases - // in the future. We'd then need a `NaturalGroupContext` - // bound to each alias to correctly disambiguate column - // references - // - // We may need to refactor `NaturalGroupContext` to become span aware in that - // case. - } - SelectItem::Except(exprs) => { - let mut indices = HashSet::new(); - for expr in exprs { - let bound = self.bind_expr(expr)?; - if let ExprImpl::InputRef(inner) = bound { - indices.insert(inner.index); - } else { - unreachable!(); + SelectItem::WildcardOrWithExcept(w) => { + match w { + Some(exprs) => { + let mut indices = HashSet::new(); + for expr in exprs { + let bound = self.bind_expr(expr)?; + if let ExprImpl::InputRef(inner) = bound { + indices.insert(inner.index); + } else { + unreachable!(); + } + } + let (exprs, names) = Self::iter_bound_columns( + self.context.columns[..] + .iter() + .filter(|c| !c.is_hidden && !indices.contains(&c.index)), + ); + select_list.extend(exprs); + aliases.extend(names); + } + None => { + if self.context.range_of.is_empty() { + return Err(ErrorCode::BindError( + "SELECT * with no tables specified is not valid".into(), + ) + .into()); + } + // Bind the column groups + // In psql, the USING and NATURAL columns come before the rest of the + // columns in a SELECT * statement + let (exprs, names) = self.iter_column_groups(); + select_list.extend(exprs); + aliases.extend(names); + + // Bind columns that are not in groups + let (exprs, names) = Self::iter_bound_columns( + self.context.columns[..].iter().filter(|c| { + !c.is_hidden + && !self + .context + .column_group_context + .mapping + .contains_key(&c.index) + }), + ); + select_list.extend(exprs); + aliases.extend(names); + // TODO: we will need to be able to handle wildcard expressions bound to + // aliases in the future. We'd then need a + // `NaturalGroupContext` bound to each alias + // to correctly disambiguate column + // references + // + // We may need to refactor `NaturalGroupContext` to become span aware in + // that case. } } - let (exprs, names) = Self::iter_bound_columns( - self.context.columns[..] - .iter() - .filter(|c| !c.is_hidden && !indices.contains(&c.index)), - ); - select_list.extend(exprs); - aliases.extend(names); } } } diff --git a/src/frontend/src/handler/create_sink.rs b/src/frontend/src/handler/create_sink.rs index 8bc2d280c531e..9132757be375a 100644 --- a/src/frontend/src/handler/create_sink.rs +++ b/src/frontend/src/handler/create_sink.rs @@ -50,7 +50,7 @@ pub fn gen_sink_query_from_name(from_name: ObjectName) -> Result { }]; let select = Select { from, - projection: vec![SelectItem::Wildcard], + projection: vec![SelectItem::WildcardOrWithExcept(None)], ..Default::default() }; let body = SetExpr::Select(Box::new(select)); diff --git a/src/meta/src/manager/catalog/utils.rs b/src/meta/src/manager/catalog/utils.rs index 03ba293f4ec9f..3d714845715d0 100644 --- a/src/meta/src/manager/catalog/utils.rs +++ b/src/meta/src/manager/catalog/utils.rs @@ -244,8 +244,8 @@ impl QueryRewriter<'_> { FunctionArgExpr::Expr(expr) | FunctionArgExpr::ExprQualifiedWildcard(expr, _) => { self.visit_expr(expr) } - FunctionArgExpr::QualifiedWildcard(_) | FunctionArgExpr::Wildcard => {} - FunctionArgExpr::Except(exprs) => { + FunctionArgExpr::QualifiedWildcard(_) | FunctionArgExpr::WildcardOrWithExcept(None) => {} + FunctionArgExpr::WildcardOrWithExcept(Some(exprs)) => { for expr in exprs { self.visit_expr(expr); } @@ -351,8 +351,8 @@ impl QueryRewriter<'_> { SelectItem::UnnamedExpr(expr) | SelectItem::ExprQualifiedWildcard(expr, _) | SelectItem::ExprWithAlias { expr, .. } => self.visit_expr(expr), - SelectItem::QualifiedWildcard(_) | SelectItem::Wildcard => {} - SelectItem::Except(exprs) => { + SelectItem::QualifiedWildcard(_) | SelectItem::WildcardOrWithExcept(None) => {} + SelectItem::WildcardOrWithExcept(Some(exprs)) => { for expr in exprs { self.visit_expr(expr); } diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs index ec3c1df1f93c6..123644d94d8f7 100644 --- a/src/sqlparser/src/ast/mod.rs +++ b/src/sqlparser/src/ast/mod.rs @@ -1917,9 +1917,8 @@ pub enum FunctionArgExpr { ExprQualifiedWildcard(Expr, Vec), /// Qualified wildcard, e.g. `alias.*` or `schema.table.*`. QualifiedWildcard(ObjectName), - /// An unqualified `*` - Wildcard, - Except(Vec), + /// An unqualified `*` or `* with (columns)` + WildcardOrWithExcept(Option>), } impl fmt::Display for FunctionArgExpr { @@ -1937,9 +1936,8 @@ impl fmt::Display for FunctionArgExpr { ) } FunctionArgExpr::QualifiedWildcard(prefix) => write!(f, "{}.*", prefix), - FunctionArgExpr::Wildcard => f.write_str("*"), - FunctionArgExpr::Except(exprs) => { - write!( + FunctionArgExpr::WildcardOrWithExcept(w) => match w { + Some(exprs) => write!( f, "EXCEPT ({})", exprs @@ -1948,8 +1946,9 @@ impl fmt::Display for FunctionArgExpr { .collect::>() .as_slice() .join(", ") - ) - } + ), + None => f.write_str("*"), + }, } } } diff --git a/src/sqlparser/src/ast/query.rs b/src/sqlparser/src/ast/query.rs index b3dfd114f126b..24b6bd1f2b41d 100644 --- a/src/sqlparser/src/ast/query.rs +++ b/src/sqlparser/src/ast/query.rs @@ -309,10 +309,8 @@ pub enum SelectItem { ExprWithAlias { expr: Expr, alias: Ident }, /// `alias.*` or even `schema.table.*` QualifiedWildcard(ObjectName), - /// An unqualified `*` - Wildcard, - /// select * except ( test ) - Except(Vec), + /// An unqualified `*`, or `* except (exprs)` + WildcardOrWithExcept(Option>), } impl fmt::Display for SelectItem { @@ -329,18 +327,20 @@ impl fmt::Display for SelectItem { .format_with("", |i, f| f(&format_args!(".{i}"))) ), SelectItem::QualifiedWildcard(prefix) => write!(f, "{}.*", prefix), - SelectItem::Wildcard => write!(f, "*"), - SelectItem::Except(exprs) => { - write!( - f, - "* EXCEPT ({})", - exprs - .iter() - .map(|v| v.to_string()) - .collect::>() - .as_slice() - .join(", ") - ) + SelectItem::WildcardOrWithExcept(w) => { + match w { + Some(exprs) => write!( + f, + "* EXCEPT ({})", + exprs + .iter() + .map(|v| v.to_string()) + .collect::>() + .as_slice() + .join(", ") + ), + None => write!(f, "*"), + } } } } diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs index 99c46d8df67e2..499ff2892ff90 100644 --- a/src/sqlparser/src/parser.rs +++ b/src/sqlparser/src/parser.rs @@ -82,8 +82,8 @@ pub enum WildcardOrExpr { /// See also [`Expr::FieldIdentifier`] for behaviors of parentheses. ExprQualifiedWildcard(Expr, Vec), QualifiedWildcard(ObjectName), - Wildcard, - Except(Vec), + // Either it's `*` or `* excepts (columns)` + WildcardOrWithExcept(Option>), } impl From for FunctionArgExpr { @@ -94,8 +94,7 @@ impl From for FunctionArgExpr { Self::ExprQualifiedWildcard(expr, prefix) } WildcardOrExpr::QualifiedWildcard(prefix) => Self::QualifiedWildcard(prefix), - WildcardOrExpr::Wildcard => Self::Wildcard, - WildcardOrExpr::Except(exprs) => Self::Except(exprs), + WildcardOrExpr::WildcardOrWithExcept(w) => Self::WildcardOrWithExcept(w), } } } @@ -318,10 +317,10 @@ impl Parser { if self.parse_keyword(Keyword::EXCEPT) && self.consume_token(&Token::LParen) { let exprs = self.parse_comma_separated(Parser::parse_expr)?; if self.consume_token(&Token::RParen) { - return Ok(WildcardOrExpr::Except(exprs)); + return Ok(WildcardOrExpr::WildcardOrWithExcept(Some(exprs))); } } else { - return Ok(WildcardOrExpr::Wildcard); + return Ok(WildcardOrExpr::WildcardOrWithExcept(None)); } } // parses wildcard field selection expression. @@ -355,10 +354,10 @@ impl Parser { let mut idents = vec![ident]; match simple_wildcard_expr { WildcardOrExpr::QualifiedWildcard(ids) => idents.extend(ids.0), - WildcardOrExpr::Wildcard => {} + WildcardOrExpr::WildcardOrWithExcept(None) => {} WildcardOrExpr::ExprQualifiedWildcard(_, _) => unreachable!(), WildcardOrExpr::Expr(e) => return Ok(WildcardOrExpr::Expr(e)), - WildcardOrExpr::Except(_) => unreachable!(), + WildcardOrExpr::WildcardOrWithExcept(Some(_)) => unreachable!(), } Ok(WildcardOrExpr::QualifiedWildcard(ObjectName(idents))) } @@ -397,10 +396,10 @@ impl Parser { match simple_wildcard_expr { WildcardOrExpr::QualifiedWildcard(ids) => idents.extend(ids.0), - WildcardOrExpr::Wildcard => {} + WildcardOrExpr::WildcardOrWithExcept(None) => {} WildcardOrExpr::ExprQualifiedWildcard(_, _) => unreachable!(), WildcardOrExpr::Expr(_) => unreachable!(), - WildcardOrExpr::Except(_) => unreachable!(), + WildcardOrExpr::WildcardOrWithExcept(Some(_)) => unreachable!(), } Ok(WildcardOrExpr::ExprQualifiedWildcard(expr, idents)) } @@ -419,7 +418,7 @@ impl Parser { Token::Word(w) => id_parts.push(w.to_ident()?), Token::Mul => { return if id_parts.is_empty() { - Ok(WildcardOrExpr::Wildcard) + Ok(WildcardOrExpr::WildcardOrWithExcept(None)) } else { Ok(WildcardOrExpr::QualifiedWildcard(ObjectName(id_parts))) } @@ -4264,8 +4263,7 @@ impl Parser { WildcardOrExpr::ExprQualifiedWildcard(expr, prefix) => { Ok(SelectItem::ExprQualifiedWildcard(expr, prefix)) } - WildcardOrExpr::Wildcard => Ok(SelectItem::Wildcard), - WildcardOrExpr::Except(exprs) => Ok(SelectItem::Except(exprs)), + WildcardOrExpr::WildcardOrWithExcept(w) => Ok(SelectItem::WildcardOrWithExcept(w)), } } diff --git a/src/tests/sqlsmith/src/sql_gen/query.rs b/src/tests/sqlsmith/src/sql_gen/query.rs index d34e24a3b078c..630ca31edeace 100644 --- a/src/tests/sqlsmith/src/sql_gen/query.rs +++ b/src/tests/sqlsmith/src/sql_gen/query.rs @@ -16,6 +16,7 @@ //! We construct Query based on the AST representation, //! as defined in the [`risingwave_sqlparser`] module. +use std::task::Context; use std::vec; use itertools::Itertools; @@ -194,9 +195,48 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { fn gen_select_list(&mut self, num_select_items: usize) -> (Vec, Vec) { let can_agg = self.flip_coin(); let context = SqlGeneratorContext::new_with_can_agg(can_agg); - (0..num_select_items) - .map(|i| self.gen_select_item(i, context)) - .unzip() + let mut i = 0; + let mut select_items = vec![]; + let mut columns = vec![]; + while i < num_select_items { + match self.rng.gen_bool(0.5) { + true => { + let (select_item, column) = self.gen_select_item(i, context); + select_items.push(select_item); + columns.push(column); + i += 1; + } + false => { + let num_included_items = cmp::min(self.rng.gen_range(1..=self.bound_columns.len()), num_select_items - i); + let (select_item, select_columns) = self.gen_select_except_item(num_included_items); + select_items.push(select_item); + columns.extend(select_columns); + i += num_included_items; + } + } + } + (select_items, columns) + } + + fn gen_select_except_item(&mut self, num_included_items: usize) -> (SelectItem, Vec) { + let mut mask = vec![1; num_included_items]; + mask.append(&mut vec![ + 0; + self.bound_relations.len() - num_included_items + ]); + mask.shuffle(self.rng); + let columns: Vec = self + .bound_columns + .iter() + .enumerate() + .filter(|(&i, _)| mask[i]) + .map(|(_, e)| e) + .collect(); + let exprs: Vec = columns + .iter() + .map(|col| Expr::Identifier(Ident::new_unchecked(col.name))) + .collect(); + (SelectItem::WildcardOrWithExcept(Some(exprs)), columns) } fn gen_select_item(&mut self, i: usize, context: SqlGeneratorContext) -> (SelectItem, Column) { From e82639e5d1242a6b73ac36f6f0a61bf18a1ad2c4 Mon Sep 17 00:00:00 2001 From: wu Date: Sun, 25 Jun 2023 02:25:09 +0000 Subject: [PATCH 11/17] update --- src/sqlparser/tests/testdata/select.yaml | 5 +++-- src/tests/sqlsmith/src/sql_gen/agg.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/sqlparser/tests/testdata/select.yaml b/src/sqlparser/tests/testdata/select.yaml index 044d48f7d857c..17e70a03506d9 100644 --- a/src/sqlparser/tests/testdata/select.yaml +++ b/src/sqlparser/tests/testdata/select.yaml @@ -29,16 +29,17 @@ formatted_sql: SELECT (CAST(ROW(1, 2, 3) AS foo)).v1.* - input: SELECT * FROM generate_series('2'::INT,'10'::INT,'2'::INT) formatted_sql: SELECT * FROM generate_series(CAST('2' AS INT), CAST('10' AS INT), CAST('2' AS INT)) - formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [Wildcard], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "generate_series", quote_style: None }]), alias: None, args: [Unnamed(Expr(Cast { expr: Value(SingleQuotedString("2")), data_type: Int })), Unnamed(Expr(Cast { expr: Value(SingleQuotedString("10")), data_type: Int })), Unnamed(Expr(Cast { expr: Value(SingleQuotedString("2")), data_type: Int }))] }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' + formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [WildcardOrWithExcept(None)], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "generate_series", quote_style: None }]), alias: None, args: [Unnamed(Expr(Cast { expr: Value(SingleQuotedString("2")), data_type: Int })), Unnamed(Expr(Cast { expr: Value(SingleQuotedString("10")), data_type: Int })), Unnamed(Expr(Cast { expr: Value(SingleQuotedString("2")), data_type: Int }))] }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' - input: SELECT * FROM unnest(Array[1,2,3]); formatted_sql: SELECT * FROM unnest(ARRAY[1, 2, 3]) - formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [Wildcard], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "unnest", quote_style: None }]), alias: None, args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("3"))], named: true })))] }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' + formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [WildcardOrWithExcept(None)], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "unnest", quote_style: None }]), alias: None, args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("3"))], named: true })))] }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' - input: SELECT id, fname, lname FROM customer WHERE salary <> 'Not Provided' AND salary <> '' formatted_sql: SELECT id, fname, lname FROM customer WHERE salary <> 'Not Provided' AND salary <> '' - input: SELECT id FROM customer WHERE NOT salary = '' formatted_sql: SELECT id FROM customer WHERE NOT salary = '' - input: SELECT * EXCEPT (v1,v2) FROM foo formatted_sql: SELECT * EXCEPT (v1, v2) FROM foo + formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [WildcardOrWithExcept(Some([Identifier(Ident { value: "v1", quote_style: None }), Identifier(Ident { value: "v2", quote_style: None })]))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "foo", quote_style: None }]), alias: None, for_system_time_as_of_proctime: false }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' - input: SELECT * FROM t LIMIT 1 FETCH FIRST ROWS ONLY error_msg: 'sql parser error: Cannot specify both LIMIT and FETCH' - input: SELECT * FROM t FETCH FIRST ROWS WITH TIES diff --git a/src/tests/sqlsmith/src/sql_gen/agg.rs b/src/tests/sqlsmith/src/sql_gen/agg.rs index a665d250f3bc4..b9dfc828b535b 100644 --- a/src/tests/sqlsmith/src/sql_gen/agg.rs +++ b/src/tests/sqlsmith/src/sql_gen/agg.rs @@ -129,7 +129,7 @@ fn make_agg_func( let args = if exprs.is_empty() { // The only agg without args is `count`. // `select proname from pg_proc where array_length(proargtypes, 1) = 0 and prokind = 'a';` - vec![FunctionArg::Unnamed(FunctionArgExpr::Wildcard)] + vec![FunctionArg::Unnamed(FunctionArgExpr::WildcardOrWithExcept(None))] } else { exprs .iter() From 6284499ab1d579131b024ccab99f39c0549f2901 Mon Sep 17 00:00:00 2001 From: wu Date: Sun, 25 Jun 2023 03:07:23 +0000 Subject: [PATCH 12/17] more test --- .../planner_test/tests/testdata/input/basic_query.yaml | 5 +++++ .../planner_test/tests/testdata/output/basic_query.yaml | 7 +++++++ src/sqlparser/tests/testdata/select.yaml | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/src/frontend/planner_test/tests/testdata/input/basic_query.yaml b/src/frontend/planner_test/tests/testdata/input/basic_query.yaml index 87370a3100a29..97e06eefeff3d 100644 --- a/src/frontend/planner_test/tests/testdata/input/basic_query.yaml +++ b/src/frontend/planner_test/tests/testdata/input/basic_query.yaml @@ -34,6 +34,11 @@ expected_outputs: - stream_plan - batch_plan +- sql: | + create table t (v1 int, v2 int, v3 int); + select * except (v1, v2), v3 from t; + expected_outputs: + - batch_plan - name: test boolean expression common factor extraction sql: | create table t (v1 Boolean, v2 Boolean, v3 Boolean); diff --git a/src/frontend/planner_test/tests/testdata/output/basic_query.yaml b/src/frontend/planner_test/tests/testdata/output/basic_query.yaml index f271321cc0ce0..0691d6d3f29e4 100644 --- a/src/frontend/planner_test/tests/testdata/output/basic_query.yaml +++ b/src/frontend/planner_test/tests/testdata/output/basic_query.yaml @@ -51,6 +51,13 @@ stream_plan: | StreamMaterialize { columns: [v3, t._row_id(hidden)], stream_key: [t._row_id], pk_columns: [t._row_id], pk_conflict: "NoCheck" } └─StreamTableScan { table: t, columns: [t.v3, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } +- sql: | + create table t (v1 int, v2 int, v3 int); + select * except (v1, v2), v3 from t; + batch_plan: | + BatchExchange { order: [], dist: Single } + └─BatchProject { exprs: [t.v3, t.v3] } + └─BatchScan { table: t, columns: [t.v3], distribution: SomeShard } - name: test boolean expression common factor extraction sql: | create table t (v1 Boolean, v2 Boolean, v3 Boolean); diff --git a/src/sqlparser/tests/testdata/select.yaml b/src/sqlparser/tests/testdata/select.yaml index 17e70a03506d9..b9e362986548d 100644 --- a/src/sqlparser/tests/testdata/select.yaml +++ b/src/sqlparser/tests/testdata/select.yaml @@ -40,6 +40,10 @@ - input: SELECT * EXCEPT (v1,v2) FROM foo formatted_sql: SELECT * EXCEPT (v1, v2) FROM foo formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [WildcardOrWithExcept(Some([Identifier(Ident { value: "v1", quote_style: None }), Identifier(Ident { value: "v2", quote_style: None })]))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "foo", quote_style: None }]), alias: None, for_system_time_as_of_proctime: false }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })' +- input: SELECT v3 EXCEPT (v1, v2) FROM foo + error_msg: |- + sql parser error: Expected SELECT, VALUES, or a subquery in the query body, found: v1 at line:1, column:21 + Near "SELECT v3 EXCEPT (" - input: SELECT * FROM t LIMIT 1 FETCH FIRST ROWS ONLY error_msg: 'sql parser error: Cannot specify both LIMIT and FETCH' - input: SELECT * FROM t FETCH FIRST ROWS WITH TIES From 6700bdb0c99f393197dd9985928c036e253326dd Mon Sep 17 00:00:00 2001 From: wu Date: Sun, 25 Jun 2023 03:07:35 +0000 Subject: [PATCH 13/17] clean code --- src/meta/src/manager/catalog/utils.rs | 3 +- src/sqlparser/src/ast/query.rs | 28 +++++++-------- src/sqlparser/tests/sqlparser_common.rs | 23 +++++++++++-- src/tests/sqlsmith/src/sql_gen/agg.rs | 4 ++- src/tests/sqlsmith/src/sql_gen/query.rs | 45 ++----------------------- 5 files changed, 41 insertions(+), 62 deletions(-) diff --git a/src/meta/src/manager/catalog/utils.rs b/src/meta/src/manager/catalog/utils.rs index 3d714845715d0..a9ce497eff6e4 100644 --- a/src/meta/src/manager/catalog/utils.rs +++ b/src/meta/src/manager/catalog/utils.rs @@ -244,7 +244,8 @@ impl QueryRewriter<'_> { FunctionArgExpr::Expr(expr) | FunctionArgExpr::ExprQualifiedWildcard(expr, _) => { self.visit_expr(expr) } - FunctionArgExpr::QualifiedWildcard(_) | FunctionArgExpr::WildcardOrWithExcept(None) => {} + FunctionArgExpr::QualifiedWildcard(_) + | FunctionArgExpr::WildcardOrWithExcept(None) => {} FunctionArgExpr::WildcardOrWithExcept(Some(exprs)) => { for expr in exprs { self.visit_expr(expr); diff --git a/src/sqlparser/src/ast/query.rs b/src/sqlparser/src/ast/query.rs index 24b6bd1f2b41d..f7d222fbb1f35 100644 --- a/src/sqlparser/src/ast/query.rs +++ b/src/sqlparser/src/ast/query.rs @@ -327,21 +327,19 @@ impl fmt::Display for SelectItem { .format_with("", |i, f| f(&format_args!(".{i}"))) ), SelectItem::QualifiedWildcard(prefix) => write!(f, "{}.*", prefix), - SelectItem::WildcardOrWithExcept(w) => { - match w { - Some(exprs) => write!( - f, - "* EXCEPT ({})", - exprs - .iter() - .map(|v| v.to_string()) - .collect::>() - .as_slice() - .join(", ") - ), - None => write!(f, "*"), - } - } + SelectItem::WildcardOrWithExcept(w) => match w { + Some(exprs) => write!( + f, + "* EXCEPT ({})", + exprs + .iter() + .map(|v| v.to_string()) + .collect::>() + .as_slice() + .join(", ") + ), + None => write!(f, "*"), + }, } } } diff --git a/src/sqlparser/tests/sqlparser_common.rs b/src/sqlparser/tests/sqlparser_common.rs index 7088450220038..615ff8d58cfd5 100644 --- a/src/sqlparser/tests/sqlparser_common.rs +++ b/src/sqlparser/tests/sqlparser_common.rs @@ -260,7 +260,10 @@ fn parse_select_all_distinct() { fn parse_select_wildcard() { let sql = "SELECT * FROM foo"; let select = verified_only_select(sql); - assert_eq!(&SelectItem::Wildcard, only(&select.projection)); + assert_eq!( + &SelectItem::WildcardOrWithExcept(None), + only(&select.projection) + ); let sql = "SELECT foo.* FROM foo"; let select = verified_only_select(sql); @@ -284,6 +287,16 @@ fn parse_select_wildcard() { assert!(format!("{}", result.unwrap_err()).contains("Expected end of statement, found: +")); } +#[test] +fn parse_select_except() { + let sql = "SELEC * EXCEPT (v1) FROM foo"; + let select = verified_only_select(sql); + assert_eq!( + &SelectItem::WildcardOrWithExcept(Some(vec![Expr::Identifier(Ident::new_unchecked("v1"))])), + only(&select.projection) + ); +} + #[test] fn parse_count_wildcard() { verified_only_select("SELECT COUNT(*) FROM Orders WHERE id = 10"); @@ -331,7 +344,9 @@ fn parse_select_count_wildcard() { assert_eq!( &Expr::Function(Function { name: ObjectName(vec![Ident::new_unchecked("COUNT")]), - args: vec![FunctionArg::Unnamed(FunctionArgExpr::Wildcard)], + args: vec![FunctionArg::Unnamed(FunctionArgExpr::WildcardOrWithExcept( + None + ))], over: None, distinct: false, order_by: vec![], @@ -1071,7 +1086,9 @@ fn parse_select_having() { Some(Expr::BinaryOp { left: Box::new(Expr::Function(Function { name: ObjectName(vec![Ident::new_unchecked("COUNT")]), - args: vec![FunctionArg::Unnamed(FunctionArgExpr::Wildcard)], + args: vec![FunctionArg::Unnamed(FunctionArgExpr::WildcardOrWithExcept( + None + ))], over: None, distinct: false, order_by: vec![], diff --git a/src/tests/sqlsmith/src/sql_gen/agg.rs b/src/tests/sqlsmith/src/sql_gen/agg.rs index b9dfc828b535b..a05c6f75ee9de 100644 --- a/src/tests/sqlsmith/src/sql_gen/agg.rs +++ b/src/tests/sqlsmith/src/sql_gen/agg.rs @@ -129,7 +129,9 @@ fn make_agg_func( let args = if exprs.is_empty() { // The only agg without args is `count`. // `select proname from pg_proc where array_length(proargtypes, 1) = 0 and prokind = 'a';` - vec![FunctionArg::Unnamed(FunctionArgExpr::WildcardOrWithExcept(None))] + vec![FunctionArg::Unnamed(FunctionArgExpr::WildcardOrWithExcept( + None, + ))] } else { exprs .iter() diff --git a/src/tests/sqlsmith/src/sql_gen/query.rs b/src/tests/sqlsmith/src/sql_gen/query.rs index 46713b1530ff8..d0b0bd3b4d1c3 100644 --- a/src/tests/sqlsmith/src/sql_gen/query.rs +++ b/src/tests/sqlsmith/src/sql_gen/query.rs @@ -197,48 +197,9 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { fn gen_select_list(&mut self, num_select_items: usize) -> (Vec, Vec) { let can_agg = self.flip_coin(); let context = SqlGeneratorContext::new_with_can_agg(can_agg); - let mut i = 0; - let mut select_items = vec![]; - let mut columns = vec![]; - while i < num_select_items { - match self.rng.gen_bool(0.5) { - true => { - let (select_item, column) = self.gen_select_item(i, context); - select_items.push(select_item); - columns.push(column); - i += 1; - } - false => { - let num_included_items = cmp::min(self.rng.gen_range(1..=self.bound_columns.len()), num_select_items - i); - let (select_item, select_columns) = self.gen_select_except_item(num_included_items); - select_items.push(select_item); - columns.extend(select_columns); - i += num_included_items; - } - } - } - (select_items, columns) - } - - fn gen_select_except_item(&mut self, num_included_items: usize) -> (SelectItem, Vec) { - let mut mask = vec![1; num_included_items]; - mask.append(&mut vec![ - 0; - self.bound_relations.len() - num_included_items - ]); - mask.shuffle(self.rng); - let columns: Vec = self - .bound_columns - .iter() - .enumerate() - .filter(|(&i, _)| mask[i]) - .map(|(_, e)| e) - .collect(); - let exprs: Vec = columns - .iter() - .map(|col| Expr::Identifier(Ident::new_unchecked(col.name))) - .collect(); - (SelectItem::WildcardOrWithExcept(Some(exprs)), columns) + (0..num_select_items) + .map(|i| self.gen_select_item(i, context)) + .unzip() } fn gen_select_item(&mut self, i: usize, context: SqlGeneratorContext) -> (SelectItem, Column) { From 2bca994e6e855a3d1c5012ff4cfc89a454efa1b4 Mon Sep 17 00:00:00 2001 From: wu Date: Sun, 25 Jun 2023 03:08:59 +0000 Subject: [PATCH 14/17] clean code --- src/tests/sqlsmith/src/sql_gen/query.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/sqlsmith/src/sql_gen/query.rs b/src/tests/sqlsmith/src/sql_gen/query.rs index d0b0bd3b4d1c3..3571911149a60 100644 --- a/src/tests/sqlsmith/src/sql_gen/query.rs +++ b/src/tests/sqlsmith/src/sql_gen/query.rs @@ -16,7 +16,6 @@ //! We construct Query based on the AST representation, //! as defined in the [`risingwave_sqlparser`] module. -use std::task::Context; use std::vec; use itertools::Itertools; From be0e24dba9e1476a73114e4a38fbae4a0dfdd022 Mon Sep 17 00:00:00 2001 From: wu Date: Sun, 25 Jun 2023 04:06:45 +0000 Subject: [PATCH 15/17] unit test --- src/sqlparser/tests/sqlparser_common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlparser/tests/sqlparser_common.rs b/src/sqlparser/tests/sqlparser_common.rs index 615ff8d58cfd5..7e56ea649c8e5 100644 --- a/src/sqlparser/tests/sqlparser_common.rs +++ b/src/sqlparser/tests/sqlparser_common.rs @@ -289,7 +289,7 @@ fn parse_select_wildcard() { #[test] fn parse_select_except() { - let sql = "SELEC * EXCEPT (v1) FROM foo"; + let sql = "SELECT * EXCEPT (v1) FROM foo"; let select = verified_only_select(sql); assert_eq!( &SelectItem::WildcardOrWithExcept(Some(vec![Expr::Identifier(Ident::new_unchecked("v1"))])), From 66da86d1302dd576d5e35f259670be49af605de2 Mon Sep 17 00:00:00 2001 From: wu Date: Sun, 25 Jun 2023 05:16:52 +0000 Subject: [PATCH 16/17] reuse common logic of wildcard --- src/frontend/src/binder/select.rs | 98 ++++++++++++++----------------- 1 file changed, 45 insertions(+), 53 deletions(-) diff --git a/src/frontend/src/binder/select.rs b/src/frontend/src/binder/select.rs index ed26ff1776fc6..7dd639ba969d1 100644 --- a/src/frontend/src/binder/select.rs +++ b/src/frontend/src/binder/select.rs @@ -260,62 +260,54 @@ impl Binder { aliases.extend(names); } SelectItem::WildcardOrWithExcept(w) => { - match w { - Some(exprs) => { - let mut indices = HashSet::new(); - for expr in exprs { - let bound = self.bind_expr(expr)?; - if let ExprImpl::InputRef(inner) = bound { - indices.insert(inner.index); - } else { - unreachable!(); - } - } - let (exprs, names) = Self::iter_bound_columns( - self.context.columns[..] - .iter() - .filter(|c| !c.is_hidden && !indices.contains(&c.index)), - ); - select_list.extend(exprs); - aliases.extend(names); - } - None => { - if self.context.range_of.is_empty() { - return Err(ErrorCode::BindError( - "SELECT * with no tables specified is not valid".into(), - ) - .into()); + if self.context.range_of.is_empty() { + return Err(ErrorCode::BindError( + "SELECT * with no tables specified is not valid".into(), + ) + .into()); + } + + // Bind the column groups + // In psql, the USING and NATURAL columns come before the rest of the + // columns in a SELECT * statement + let (exprs, names) = self.iter_column_groups(); + select_list.extend(exprs); + aliases.extend(names); + + let mut except_indices: HashSet = HashSet::new(); + if let Some(exprs) = w { + for expr in exprs { + let bound = self.bind_expr(expr)?; + if let ExprImpl::InputRef(inner) = bound { + except_indices.insert(inner.index); + } else { + unreachable!(); } - // Bind the column groups - // In psql, the USING and NATURAL columns come before the rest of the - // columns in a SELECT * statement - let (exprs, names) = self.iter_column_groups(); - select_list.extend(exprs); - aliases.extend(names); - - // Bind columns that are not in groups - let (exprs, names) = Self::iter_bound_columns( - self.context.columns[..].iter().filter(|c| { - !c.is_hidden - && !self - .context - .column_group_context - .mapping - .contains_key(&c.index) - }), - ); - select_list.extend(exprs); - aliases.extend(names); - // TODO: we will need to be able to handle wildcard expressions bound to - // aliases in the future. We'd then need a - // `NaturalGroupContext` bound to each alias - // to correctly disambiguate column - // references - // - // We may need to refactor `NaturalGroupContext` to become span aware in - // that case. } } + + // Bind columns that are not in groups + let (exprs, names) = + Self::iter_bound_columns(self.context.columns[..].iter().filter(|c| { + !c.is_hidden + && !self + .context + .column_group_context + .mapping + .contains_key(&c.index) + && !except_indices.contains(&c.index) + })); + + select_list.extend(exprs); + aliases.extend(names); + // TODO: we will need to be able to handle wildcard expressions bound to + // aliases in the future. We'd then need a + // `NaturalGroupContext` bound to each alias + // to correctly disambiguate column + // references + // + // We may need to refactor `NaturalGroupContext` to become span aware in + // that case. } } } From 0bfaa3b54857184122c7daeb93ed24c394998490 Mon Sep 17 00:00:00 2001 From: wu Date: Sun, 25 Jun 2023 07:27:25 +0000 Subject: [PATCH 17/17] fix new format --- .../planner_test/tests/testdata/output/basic_query.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/frontend/planner_test/tests/testdata/output/basic_query.yaml b/src/frontend/planner_test/tests/testdata/output/basic_query.yaml index a5e8d3f9fd183..81bf434ba9b09 100644 --- a/src/frontend/planner_test/tests/testdata/output/basic_query.yaml +++ b/src/frontend/planner_test/tests/testdata/output/basic_query.yaml @@ -44,16 +44,16 @@ - sql: | create table t (v1 int, v2 int, v3 int); select * except (v1, v2) from t; - batch_plan: | + batch_plan: |- BatchExchange { order: [], dist: Single } └─BatchScan { table: t, columns: [t.v3], distribution: SomeShard } - stream_plan: | - StreamMaterialize { columns: [v3, t._row_id(hidden)], stream_key: [t._row_id], pk_columns: [t._row_id], pk_conflict: "NoCheck" } + stream_plan: |- + StreamMaterialize { columns: [v3, t._row_id(hidden)], stream_key: [t._row_id], pk_columns: [t._row_id], pk_conflict: NoCheck } └─StreamTableScan { table: t, columns: [t.v3, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) } - sql: | create table t (v1 int, v2 int, v3 int); select * except (v1, v2), v3 from t; - batch_plan: | + batch_plan: |- BatchExchange { order: [], dist: Single } └─BatchProject { exprs: [t.v3, t.v3] } └─BatchScan { table: t, columns: [t.v3], distribution: SomeShard }