From 18f8b84601d091b7cc8b9bd2469b05c83a092e05 Mon Sep 17 00:00:00 2001 From: Guillaume Balaine Date: Sun, 22 Aug 2021 18:32:13 +0200 Subject: [PATCH 1/8] enable integer keys for map access --- src/parser.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/parser.rs b/src/parser.rs index 63bbc8ffd..40173f011 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -951,7 +951,7 @@ impl<'a> Parser<'a> { } pub fn parse_map_access(&mut self, expr: Expr) -> Result { - let key = self.parse_literal_string()?; + let key = self.parse_map_key_string()?; let tok = self.consume_token(&Token::RBracket); debug!("Tok: {}", tok); match expr { @@ -1995,6 +1995,16 @@ impl<'a> Parser<'a> { } } + /// Parse a map key string + pub fn parse_map_key_string(&mut self) -> Result { + match self.next_token() { + Token::Word(Word { value, keyword, .. }) if keyword == Keyword::NoKeyword => Ok(value), + Token::SingleQuotedString(s) => Ok(s), + Token::Number(s, _) => Ok(s), + unexpected => self.expected("literal string or number", unexpected), + } + } + /// Parse a SQL datatype (in the context of a CREATE TABLE statement for example) pub fn parse_data_type(&mut self) -> Result { match self.next_token() { From 40de9d6533bc158d5e3bd1ecc869660f1daff967 Mon Sep 17 00:00:00 2001 From: Guillaume Balaine Date: Tue, 24 Aug 2021 13:58:50 +0200 Subject: [PATCH 2/8] enable map access for number keys --- src/ast/mod.rs | 12 ++++++++++-- src/parser.rs | 22 ++++++++++++++++------ tests/sqlparser_common.rs | 1 + tests/sqlparser_postgres.rs | 23 +++++++++++++++++++++++ 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 6aeb00685..f7d384b7b 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -251,7 +251,7 @@ pub enum Expr { }, MapAccess { column: Box, - key: String, + keys: Vec, }, /// Scalar function call e.g. `LEFT(foo, 5)` Function(Function), @@ -280,7 +280,15 @@ impl fmt::Display for Expr { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Expr::Identifier(s) => write!(f, "{}", s), - Expr::MapAccess { column, key } => write!(f, "{}[\"{}\"]", column, key), + Expr::MapAccess { column, keys } => { + write!(f, "{}{}", column, keys.into_iter().map(|k| { + match k { + k @ Value::Number(_, _) => format!("[{}]", k), + _ => format!("[\"{}\"]", k) + } + + }).collect::>().join("")) + }, Expr::Wildcard => f.write_str("*"), Expr::QualifiedWildcard(q) => write!(f, "{}.*", display_separated(q, ".")), Expr::CompoundIdentifier(s) => write!(f, "{}", display_separated(s, ".")), diff --git a/src/parser.rs b/src/parser.rs index 40173f011..ace560dc6 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -951,13 +951,20 @@ impl<'a> Parser<'a> { } pub fn parse_map_access(&mut self, expr: Expr) -> Result { - let key = self.parse_map_key_string()?; + let key = self.parse_map_key()?; let tok = self.consume_token(&Token::RBracket); debug!("Tok: {}", tok); + let mut key_parts: Vec = vec![key]; + while self.consume_token(&Token::LBracket) { + let key = self.parse_map_key()?; + let tok = self.consume_token(&Token::RBracket); + debug!("Tok: {}", tok); + key_parts.push(key); + } match expr { e @ Expr::Identifier(_) | e @ Expr::CompoundIdentifier(_) => Ok(Expr::MapAccess { column: Box::new(e), - key, + keys: key_parts, }), _ => Ok(expr), } @@ -1996,11 +2003,14 @@ impl<'a> Parser<'a> { } /// Parse a map key string - pub fn parse_map_key_string(&mut self) -> Result { + pub fn parse_map_key(&mut self) -> Result { match self.next_token() { - Token::Word(Word { value, keyword, .. }) if keyword == Keyword::NoKeyword => Ok(value), - Token::SingleQuotedString(s) => Ok(s), - Token::Number(s, _) => Ok(s), + Token::Word(Word { value, keyword, .. }) if keyword == Keyword::NoKeyword => Ok(Value::SingleQuotedString(value)), + Token::SingleQuotedString(s) => Ok(Value::SingleQuotedString(s)), + #[cfg(not(feature = "bigdecimal"))] + Token::Number(s, _) => Ok(Value::Number(s, false)), + #[cfg(feature = "bigdecimal")] + Token::Number(s, _) => Ok(Value::Number(s.parse().unwrap(), false)), unexpected => self.expected("literal string or number", unexpected), } } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 5bc052a34..95cfb7d3a 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -26,6 +26,7 @@ use matches::assert_matches; use sqlparser::ast::*; use sqlparser::dialect::{keywords::ALL_KEYWORDS, GenericDialect, SQLiteDialect}; use sqlparser::parser::{Parser, ParserError}; +use sqlparser::ast::Expr::Identifier; #[test] fn parse_insert_values() { diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 2e66d313b..053ade8d7 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -21,6 +21,9 @@ use test_utils::*; use sqlparser::ast::*; use sqlparser::dialect::{GenericDialect, PostgreSqlDialect}; use sqlparser::parser::ParserError; +use sqlparser::ast::Expr::{Identifier, MapAccess}; +#[cfg(feature = "bigdecimal")] +use bigdecimal::BigDecimal; #[test] fn parse_create_table_with_defaults() { @@ -669,6 +672,26 @@ fn parse_pg_regex_match_ops() { } } +#[test] +fn parse_map_access_expr() { + //let sql = "SELECT foo[0] as foozero, bar[\"baz\"] as barbaz FROM foos"; + let sql = "SELECT foo[0] FROM foos"; + let select = pg_and_generic().verified_only_select(sql); + #[cfg(not(feature = "bigdecimal"))] + assert_eq!( + &MapAccess { column: Box::new(Identifier(Ident { value: "foo".to_string(), quote_style: None })), keys: vec![Value::Number("0".to_string(), false)] }, + expr_from_projection(only(&select.projection)), + ); + let sql = "SELECT foo[0][0] FROM foos"; + let select = pg_and_generic().verified_only_select(sql); + #[cfg(not(feature = "bigdecimal"))] + assert_eq!( + &MapAccess { column: Box::new(Identifier(Ident { value: "foo".to_string(), quote_style: None })), keys: vec![Value::Number("0".to_string(), false), Value::Number("0".to_string(), false)] }, + expr_from_projection(only(&select.projection)), + ); +} + + fn pg() -> TestedDialects { TestedDialects { dialects: vec![Box::new(PostgreSqlDialect {})], From 7d599de77b4fb5bd13e801b4342f1caa80765854 Mon Sep 17 00:00:00 2001 From: Guillaume Balaine Date: Sat, 18 Sep 2021 16:07:12 +0200 Subject: [PATCH 3/8] Add tests for string based map access --- src/ast/mod.rs | 2 +- tests/sqlparser_common.rs | 1 - tests/sqlparser_postgres.rs | 8 +++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index f7d384b7b..f11d36f68 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -284,7 +284,7 @@ impl fmt::Display for Expr { write!(f, "{}{}", column, keys.into_iter().map(|k| { match k { k @ Value::Number(_, _) => format!("[{}]", k), - _ => format!("[\"{}\"]", k) + _ => format!("[{}]", k) } }).collect::>().join("")) diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 95cfb7d3a..5bc052a34 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -26,7 +26,6 @@ use matches::assert_matches; use sqlparser::ast::*; use sqlparser::dialect::{keywords::ALL_KEYWORDS, GenericDialect, SQLiteDialect}; use sqlparser::parser::{Parser, ParserError}; -use sqlparser::ast::Expr::Identifier; #[test] fn parse_insert_values() { diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 053ade8d7..07892aa88 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -674,7 +674,6 @@ fn parse_pg_regex_match_ops() { #[test] fn parse_map_access_expr() { - //let sql = "SELECT foo[0] as foozero, bar[\"baz\"] as barbaz FROM foos"; let sql = "SELECT foo[0] FROM foos"; let select = pg_and_generic().verified_only_select(sql); #[cfg(not(feature = "bigdecimal"))] @@ -689,6 +688,13 @@ fn parse_map_access_expr() { &MapAccess { column: Box::new(Identifier(Ident { value: "foo".to_string(), quote_style: None })), keys: vec![Value::Number("0".to_string(), false), Value::Number("0".to_string(), false)] }, expr_from_projection(only(&select.projection)), ); + let sql = r#"SELECT bar[0]['baz']['fooz'] FROM foos"#; + let select = pg_and_generic().verified_only_select(sql); + #[cfg(not(feature = "bigdecimal"))] + assert_eq!( + &MapAccess { column: Box::new(Identifier(Ident { value: "bar".to_string(), quote_style: None })), keys: vec![Value::Number("0".to_string(), false), Value::SingleQuotedString("baz".to_string()), Value::SingleQuotedString("fooz".to_string())] }, + expr_from_projection(only(&select.projection)), + ); } From 90072c2e2dc1aed8e64cb063cf834ef9e97b8f37 Mon Sep 17 00:00:00 2001 From: Guillaume Balaine Date: Fri, 24 Sep 2021 11:43:42 +0200 Subject: [PATCH 4/8] MapAccess: unbox single quoted strings to always display double quoted strings for map access --- src/ast/mod.rs | 1 + tests/sqlparser_hive.rs | 2 +- tests/sqlparser_postgres.rs | 17 +++++++++-------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index f11d36f68..ad0a5916d 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -284,6 +284,7 @@ impl fmt::Display for Expr { write!(f, "{}{}", column, keys.into_iter().map(|k| { match k { k @ Value::Number(_, _) => format!("[{}]", k), + Value::SingleQuotedString(s) => format!("[\"{}\"]", s), _ => format!("[{}]", k) } diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 585be989b..d933f0f25 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -194,7 +194,7 @@ fn rename_table() { #[test] fn map_access() { - let rename = "SELECT a.b[\"asdf\"] FROM db.table WHERE a = 2"; + let rename = r#"SELECT a.b["asdf"] FROM db.table WHERE a = 2"#; hive().verified_stmt(rename); } diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 07892aa88..f843a64df 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -674,30 +674,30 @@ fn parse_pg_regex_match_ops() { #[test] fn parse_map_access_expr() { + #[cfg(not(feature = "bigdecimal"))] + let zero = "0".to_string(); + #[cfg(feature = "bigdecimal")] + let zero = BigDecimal::parse_bytes(b"0", 10).unwrap(); let sql = "SELECT foo[0] FROM foos"; let select = pg_and_generic().verified_only_select(sql); - #[cfg(not(feature = "bigdecimal"))] assert_eq!( - &MapAccess { column: Box::new(Identifier(Ident { value: "foo".to_string(), quote_style: None })), keys: vec![Value::Number("0".to_string(), false)] }, + &MapAccess { column: Box::new(Identifier(Ident { value: "foo".to_string(), quote_style: None })), keys: vec![Value::Number(zero.clone(), false)] }, expr_from_projection(only(&select.projection)), ); let sql = "SELECT foo[0][0] FROM foos"; let select = pg_and_generic().verified_only_select(sql); - #[cfg(not(feature = "bigdecimal"))] assert_eq!( - &MapAccess { column: Box::new(Identifier(Ident { value: "foo".to_string(), quote_style: None })), keys: vec![Value::Number("0".to_string(), false), Value::Number("0".to_string(), false)] }, + &MapAccess { column: Box::new(Identifier(Ident { value: "foo".to_string(), quote_style: None })), keys: vec![Value::Number(zero.clone(), false), Value::Number(zero.clone(), false)] }, expr_from_projection(only(&select.projection)), ); - let sql = r#"SELECT bar[0]['baz']['fooz'] FROM foos"#; + let sql = r#"SELECT bar[0]["baz"]["fooz"] FROM foos"#; let select = pg_and_generic().verified_only_select(sql); - #[cfg(not(feature = "bigdecimal"))] assert_eq!( - &MapAccess { column: Box::new(Identifier(Ident { value: "bar".to_string(), quote_style: None })), keys: vec![Value::Number("0".to_string(), false), Value::SingleQuotedString("baz".to_string()), Value::SingleQuotedString("fooz".to_string())] }, + &MapAccess { column: Box::new(Identifier(Ident { value: "bar".to_string(), quote_style: None })), keys: vec![Value::Number(zero, false), Value::SingleQuotedString("baz".to_string()), Value::SingleQuotedString("fooz".to_string())] }, expr_from_projection(only(&select.projection)), ); } - fn pg() -> TestedDialects { TestedDialects { dialects: vec![Box::new(PostgreSqlDialect {})], @@ -709,3 +709,4 @@ fn pg_and_generic() -> TestedDialects { dialects: vec![Box::new(PostgreSqlDialect {}), Box::new(GenericDialect {})], } } + From a5bb345afae848966a1dcea488135cca6ba8f6d7 Mon Sep 17 00:00:00 2001 From: Guillaume Balaine Date: Fri, 24 Sep 2021 11:44:11 +0200 Subject: [PATCH 5/8] cargo fmt --- src/ast/mod.rs | 25 +++++++++++++++--------- src/parser.rs | 4 +++- tests/sqlparser_postgres.rs | 38 ++++++++++++++++++++++++++++++------- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index ad0a5916d..3019acbfb 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -281,15 +281,22 @@ impl fmt::Display for Expr { match self { Expr::Identifier(s) => write!(f, "{}", s), Expr::MapAccess { column, keys } => { - write!(f, "{}{}", column, keys.into_iter().map(|k| { - match k { - k @ Value::Number(_, _) => format!("[{}]", k), - Value::SingleQuotedString(s) => format!("[\"{}\"]", s), - _ => format!("[{}]", k) - } - - }).collect::>().join("")) - }, + write!( + f, + "{}{}", + column, + keys.into_iter() + .map(|k| { + match k { + k @ Value::Number(_, _) => format!("[{}]", k), + Value::SingleQuotedString(s) => format!("[\"{}\"]", s), + _ => format!("[{}]", k), + } + }) + .collect::>() + .join("") + ) + } Expr::Wildcard => f.write_str("*"), Expr::QualifiedWildcard(q) => write!(f, "{}.*", display_separated(q, ".")), Expr::CompoundIdentifier(s) => write!(f, "{}", display_separated(s, ".")), diff --git a/src/parser.rs b/src/parser.rs index ace560dc6..84a82b121 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -2005,7 +2005,9 @@ impl<'a> Parser<'a> { /// Parse a map key string pub fn parse_map_key(&mut self) -> Result { match self.next_token() { - Token::Word(Word { value, keyword, .. }) if keyword == Keyword::NoKeyword => Ok(Value::SingleQuotedString(value)), + Token::Word(Word { value, keyword, .. }) if keyword == Keyword::NoKeyword => { + Ok(Value::SingleQuotedString(value)) + } Token::SingleQuotedString(s) => Ok(Value::SingleQuotedString(s)), #[cfg(not(feature = "bigdecimal"))] Token::Number(s, _) => Ok(Value::Number(s, false)), diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index f843a64df..43baeb5a5 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -18,12 +18,12 @@ mod test_utils; use test_utils::*; +#[cfg(feature = "bigdecimal")] +use bigdecimal::BigDecimal; +use sqlparser::ast::Expr::{Identifier, MapAccess}; use sqlparser::ast::*; use sqlparser::dialect::{GenericDialect, PostgreSqlDialect}; use sqlparser::parser::ParserError; -use sqlparser::ast::Expr::{Identifier, MapAccess}; -#[cfg(feature = "bigdecimal")] -use bigdecimal::BigDecimal; #[test] fn parse_create_table_with_defaults() { @@ -681,19 +681,44 @@ fn parse_map_access_expr() { let sql = "SELECT foo[0] FROM foos"; let select = pg_and_generic().verified_only_select(sql); assert_eq!( - &MapAccess { column: Box::new(Identifier(Ident { value: "foo".to_string(), quote_style: None })), keys: vec![Value::Number(zero.clone(), false)] }, + &MapAccess { + column: Box::new(Identifier(Ident { + value: "foo".to_string(), + quote_style: None + })), + keys: vec![Value::Number(zero.clone(), false)] + }, expr_from_projection(only(&select.projection)), ); let sql = "SELECT foo[0][0] FROM foos"; let select = pg_and_generic().verified_only_select(sql); assert_eq!( - &MapAccess { column: Box::new(Identifier(Ident { value: "foo".to_string(), quote_style: None })), keys: vec![Value::Number(zero.clone(), false), Value::Number(zero.clone(), false)] }, + &MapAccess { + column: Box::new(Identifier(Ident { + value: "foo".to_string(), + quote_style: None + })), + keys: vec![ + Value::Number(zero.clone(), false), + Value::Number(zero.clone(), false) + ] + }, expr_from_projection(only(&select.projection)), ); let sql = r#"SELECT bar[0]["baz"]["fooz"] FROM foos"#; let select = pg_and_generic().verified_only_select(sql); assert_eq!( - &MapAccess { column: Box::new(Identifier(Ident { value: "bar".to_string(), quote_style: None })), keys: vec![Value::Number(zero, false), Value::SingleQuotedString("baz".to_string()), Value::SingleQuotedString("fooz".to_string())] }, + &MapAccess { + column: Box::new(Identifier(Ident { + value: "bar".to_string(), + quote_style: None + })), + keys: vec![ + Value::Number(zero, false), + Value::SingleQuotedString("baz".to_string()), + Value::SingleQuotedString("fooz".to_string()) + ] + }, expr_from_projection(only(&select.projection)), ); } @@ -709,4 +734,3 @@ fn pg_and_generic() -> TestedDialects { dialects: vec![Box::new(PostgreSqlDialect {}), Box::new(GenericDialect {})], } } - From aba4192918b111f09bddc30555289473ede35dce Mon Sep 17 00:00:00 2001 From: Guillaume Balaine Date: Fri, 24 Sep 2021 11:44:53 +0200 Subject: [PATCH 6/8] cargo clippy --- src/ast/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 3019acbfb..da6c42873 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -285,7 +285,7 @@ impl fmt::Display for Expr { f, "{}{}", column, - keys.into_iter() + keys.iter() .map(|k| { match k { k @ Value::Number(_, _) => format!("[{}]", k), From c9e3cfba47d2df923fdb287e0af7032025d3f4aa Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 24 Sep 2021 14:17:56 -0400 Subject: [PATCH 7/8] Fix compilation with nost by avoiding format! --- src/ast/mod.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index da6c42873..e9f052fd3 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -281,21 +281,15 @@ impl fmt::Display for Expr { match self { Expr::Identifier(s) => write!(f, "{}", s), Expr::MapAccess { column, keys } => { - write!( - f, - "{}{}", - column, - keys.iter() - .map(|k| { - match k { - k @ Value::Number(_, _) => format!("[{}]", k), - Value::SingleQuotedString(s) => format!("[\"{}\"]", s), - _ => format!("[{}]", k), - } - }) - .collect::>() - .join("") - ) + write!(f, "{}", column)?; + for k in keys { + match k { + k @ Value::Number(_, _) => write!(f, "[{}]", k)?, + Value::SingleQuotedString(s) => write!(f, "[\"{}\"]", s)?, + _ => write!(f, "[{}]", k)? + } + } + Ok(()) } Expr::Wildcard => f.write_str("*"), Expr::QualifiedWildcard(q) => write!(f, "{}.*", display_separated(q, ".")), From 697899162339934d29b703cd8258d9c1687cd7b4 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 24 Sep 2021 14:18:51 -0400 Subject: [PATCH 8/8] fix codestyle --- src/ast/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index e9f052fd3..95e670093 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -286,7 +286,7 @@ impl fmt::Display for Expr { match k { k @ Value::Number(_, _) => write!(f, "[{}]", k)?, Value::SingleQuotedString(s) => write!(f, "[\"{}\"]", s)?, - _ => write!(f, "[{}]", k)? + _ => write!(f, "[{}]", k)?, } } Ok(())