append sql context to parser error

Signed-off-by: Runji Wang <wangrunji0408@163.com>
risingwavelabs · May 28, 2024 · c07550a · c07550a
1 parent 68d499e
commit c07550a
Show file tree

Hide file tree

Showing 10 changed files with 209 additions and 109 deletions.
diff --git a/e2e_test/error_ui/extended/main.slt b/e2e_test/error_ui/extended/main.slt
@@ -4,8 +4,9 @@ selet 1;
 db error: ERROR: Failed to prepare the statement
 
 Caused by:
-  sql parser error: Expected an SQL statement, found: selet at line 1, column 1
-Near "selet"
+  sql parser error: expected an SQL statement, found: selet at line 1, column 1
+  LINE 1: selet 1;
+               ^
 
 
 query error

diff --git a/e2e_test/error_ui/simple/main.slt b/e2e_test/error_ui/simple/main.slt
@@ -4,9 +4,9 @@ selet 1;
 db error: ERROR: Failed to run the query
 
 Caused by:
-  sql parser error: Expected an SQL statement, found: selet at line 1, column 1
-Near "selet"
-
+  sql parser error: expected an SQL statement, found: selet at line 1, column 1
+  LINE 1: selet 1;
+               ^
 
 statement error
 create function int_42() returns int as int_42 using link '555.0.0.1:8815';

diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs
@@ -226,22 +226,57 @@ impl Parser {
         let mut tokenizer = Tokenizer::new(sql);
         let tokens = tokenizer.tokenize_with_location()?;
         let mut parser = Parser::new(tokens);
+        let ast = parser.parse_statements().map_err(|e| {
+            // append SQL context to the error message, e.g.:
+            // LINE 1: SELECT 1::int(2);
+            //                      ^
+            // XXX: the cursor location is not accurate
+            //      it may be offset one token forward because the error token has been consumed
+            let loc = match parser.tokens.get(parser.index) {
+                Some(token) => token.location.clone(),
+                None => {
+                    // get location of EOF
+                    Location {
+                        line: sql.lines().count() as u64,
+                        column: sql.lines().last().map_or(0, |l| l.len() as u64) + 1,
+                    }
+                }
+            };
+            let prefix = format!("LINE {}: ", loc.line);
+            let sql_line = sql.split('\n').nth(loc.line as usize - 1).unwrap();
+            let cursor = std::iter::repeat(' ')
+                .take(prefix.len() + loc.column as usize - 1)
+                .chain(['^'])
+                .collect::<String>();
+            ParserError::ParserError(format!(
+                "{}\n{}{}\n{}",
+                e.inner_msg(),
+                prefix,
+                sql_line,
+                cursor
+            ))
+        })?;
+        Ok(ast)
+    }
+
+    /// Parse a list of semicolon-separated SQL statements.
+    pub fn parse_statements(&mut self) -> Result<Vec<Statement>, ParserError> {
         let mut stmts = Vec::new();
         let mut expecting_statement_delimiter = false;
         loop {
             // ignore empty statements (between successive statement delimiters)
-            while parser.consume_token(&Token::SemiColon) {
+            while self.consume_token(&Token::SemiColon) {
                 expecting_statement_delimiter = false;
             }
 
-            if parser.peek_token() == Token::EOF {
+            if self.peek_token() == Token::EOF {
                 break;
             }
             if expecting_statement_delimiter {
-                return parser.expected("end of statement", parser.peek_token());
+                return self.expected("end of statement", self.peek_token());
             }
 
-            let statement = parser.parse_statement()?;
+            let statement = self.parse_statement()?;
             stmts.push(statement);
             expecting_statement_delimiter = true;
         }
@@ -1958,24 +1993,7 @@ impl Parser {
 
     /// Report unexpected token
     pub fn expected<T>(&self, expected: &str, found: TokenWithLocation) -> Result<T, ParserError> {
-        let start_off = self.index.saturating_sub(10);
-        let end_off = self.index.min(self.tokens.len());
-        let near_tokens = &self.tokens[start_off..end_off];
-        struct TokensDisplay<'a>(&'a [TokenWithLocation]);
-        impl<'a> fmt::Display for TokensDisplay<'a> {
-            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-                for token in self.0 {
-                    write!(f, "{}", token.token)?;
-                }
-                Ok(())
-            }
-        }
-        parser_err!(format!(
-            "Expected {}, found: {}\nNear \"{}\"",
-            expected,
-            found,
-            TokensDisplay(near_tokens),
-        ))
+        parser_err!(format!("expected {}, found: {}", expected, found))
     }
 
     /// Look for an expected keyword and consume it if it exists

diff --git a/src/sqlparser/tests/testdata/array.yaml b/src/sqlparser/tests/testdata/array.yaml
@@ -6,13 +6,20 @@
 - input: CREATE TABLE t(a int[][][]);
   formatted_sql: CREATE TABLE t (a INT[][][])
 - input: CREATE TABLE t(a int[);
-  error_msg: 'sql parser error: Unexpected ) at line 1, column 22'
+  error_msg: |-
+    sql parser error: Unexpected ) at line 1, column 22
+    LINE 1: CREATE TABLE t(a int[);
+                                 ^
 - input: CREATE TABLE t(a int[[]);
-  error_msg: 'sql parser error: Unexpected [ at line 1, column 22'
+  error_msg: |-
+    sql parser error: Unexpected [ at line 1, column 22
+    LINE 1: CREATE TABLE t(a int[[]);
+                                 ^
 - input: CREATE TABLE t(a int]);
   error_msg: |-
-    sql parser error: Expected ',' or ')' after column definition, found: ] at line 1, column 21
-    Near "CREATE TABLE t(a int"
+    sql parser error: expected ',' or ')' after column definition, found: ] at line 1, column 21
+    LINE 1: CREATE TABLE t(a int]);
+                                ^
 - input: SELECT foo[0] FROM foos
   formatted_sql: SELECT foo[0] FROM foos
 - input: SELECT foo[0][0] FROM foos
@@ -29,33 +36,41 @@
   formatted_sql: SELECT ARRAY[[], []]
 - input: SELECT ARRAY[ARRAY[],[]]
   error_msg: |-
-    sql parser error: Expected an expression:, found: [ at line 1, column 22
-    Near "SELECT ARRAY[ARRAY[],["
+    sql parser error: expected an expression:, found: [ at line 1, column 22
+    LINE 1: SELECT ARRAY[ARRAY[],[]]
+                                  ^
 - input: SELECT ARRAY[[],ARRAY[]]
   error_msg: |-
-    sql parser error: Expected [, found: ARRAY at line 1, column 17
-    Near "SELECT ARRAY[[],"
+    sql parser error: expected [, found: ARRAY at line 1, column 17
+    LINE 1: SELECT ARRAY[[],ARRAY[]]
+                            ^
 - input: SELECT ARRAY[[1,2],3]
   error_msg: |-
-    sql parser error: Expected [, found: 3 at line 1, column 20
-    Near "SELECT ARRAY[[1,2],"
+    sql parser error: expected [, found: 3 at line 1, column 20
+    LINE 1: SELECT ARRAY[[1,2],3]
+                               ^
 - input: SELECT ARRAY[1,[2,3]]
   error_msg: |-
-    sql parser error: Expected an expression:, found: [ at line 1, column 16
-    Near "SELECT ARRAY[1,["
+    sql parser error: expected an expression:, found: [ at line 1, column 16
+    LINE 1: SELECT ARRAY[1,[2,3]]
+                            ^
 - input: SELECT ARRAY[ARRAY[1,2],[3,4]]
   error_msg: |-
-    sql parser error: Expected an expression:, found: [ at line 1, column 25
-    Near "ARRAY[ARRAY[1,2],["
+    sql parser error: expected an expression:, found: [ at line 1, column 25
+    LINE 1: SELECT ARRAY[ARRAY[1,2],[3,4]]
+                                     ^
 - input: SELECT ARRAY[[1,2],ARRAY[3,4]]
   error_msg: |-
-    sql parser error: Expected [, found: ARRAY at line 1, column 20
-    Near "SELECT ARRAY[[1,2],"
+    sql parser error: expected [, found: ARRAY at line 1, column 20
+    LINE 1: SELECT ARRAY[[1,2],ARRAY[3,4]]
+                               ^
 - input: SELECT ARRAY[[1,2],[3] || [4]]
   error_msg: |-
-    sql parser error: Expected ], found: || at line 1, column 24
-    Near "[[1,2],[3]"
+    sql parser error: expected ], found: || at line 1, column 24
+    LINE 1: SELECT ARRAY[[1,2],[3] || [4]]
+                                  ^
 - input: SELECT [1,2]
   error_msg: |-
-    sql parser error: Expected an expression:, found: [ at line 1, column 8
-    Near "SELECT ["
+    sql parser error: expected an expression:, found: [ at line 1, column 8
+    LINE 1: SELECT [1,2]
+                    ^
diff --git a/src/sqlparser/tests/testdata/create.yaml b/src/sqlparser/tests/testdata/create.yaml
@@ -17,12 +17,14 @@
   formatted_sql: CREATE TABLE t (a INT, b INT) AS SELECT 1 AS b, 2 AS a
 - input: CREATE SOURCE src
   error_msg: |-
-    sql parser error: Expected description of the format, found: EOF at the end
-    Near "CREATE SOURCE src"
+    sql parser error: expected description of the format, found: EOF at the end
+    LINE 1: CREATE SOURCE src
+                             ^
 - input: CREATE SOURCE src-a FORMAT PLAIN ENCODE JSON
   error_msg: |-
-    sql parser error: Expected description of the format, found: - at line 1, column 18
-    Near "CREATE SOURCE src"
+    sql parser error: expected description of the format, found: - at line 1, column 18
+    LINE 1: CREATE SOURCE src-a FORMAT PLAIN ENCODE JSON
+                             ^
 - input: CREATE SOURCE src FORMAT PLAIN ENCODE JSON
   formatted_sql: CREATE SOURCE src FORMAT PLAIN ENCODE JSON
 - input: CREATE SOURCE mysql_src with ( connector = 'mysql-cdc', hostname = 'localhost', port = '3306', database.name = 'mytest', server.id = '5601' )
@@ -31,8 +33,9 @@
   formatted_sql: CREATE TABLE sbtest10 (id INT PRIMARY KEY, k INT, c CHARACTER VARYING, pad CHARACTER VARYING) FROM sbtest TABLE 'mydb.sbtest10'
 - input: CREATE TABLE sbtest10 (id INT PRIMARY KEY, k INT, c CHARACTER VARYING, pad CHARACTER VARYING) FROM sbtest
   error_msg: |-
-    sql parser error: Expected TABLE, found: EOF at the end
-    Near "pad CHARACTER VARYING) FROM sbtest"
+    sql parser error: expected TABLE, found: EOF at the end
+    LINE 1: CREATE TABLE sbtest10 (id INT PRIMARY KEY, k INT, c CHARACTER VARYING, pad CHARACTER VARYING) FROM sbtest
+                                                                                                                     ^
 - input: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.servers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.location = 'file://')
   formatted_sql: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.servers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.location = 'file://')
   formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "servers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "location", quote_style: None }]), value: SingleQuotedString("file://") }], key_encode: None }), source_watermarks: [], include_column_options: [] } }'
@@ -49,7 +52,10 @@
 - input: CREATE TABLE T (a STRUCT<v1 INT>)
   formatted_sql: CREATE TABLE T (a STRUCT<v1 INT>)
 - input: CREATE TABLE T (FULL INT)
-  error_msg: 'sql parser error: syntax error at or near FULL at line 1, column 17'
+  error_msg: |-
+    sql parser error: syntax error at or near FULL at line 1, column 17
+    LINE 1: CREATE TABLE T (FULL INT)
+                                ^
 - input: CREATE TABLE T ("FULL" INT)
   formatted_sql: CREATE TABLE T ("FULL" INT)
 - input: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMN("FULL")
@@ -58,22 +64,28 @@
   formatted_sql: CREATE TABLE T ("FULL" INT) ON CONFLICT IGNORE
 - input: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMN
   error_msg: |-
-    sql parser error: Expected (, found: EOF at the end
-    Near " CONFLICT OVERWRITE WITH VERSION COLUMN"
+    sql parser error: expected (, found: EOF at the end
+    LINE 1: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMN
+                                                                                 ^
 - input: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMN(FULL
-  error_msg: 'sql parser error: syntax error at or near FULL at line 1, column 71'
+  error_msg: |-
+    sql parser error: syntax error at or near FULL at line 1, column 71
+    LINE 1: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMN(FULL
+                                                                                      ^
 - input: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMNFULL)
   error_msg: |-
-    sql parser error: Expected (, found: VERSION at line 1, column 56
-    Near "INT) ON CONFLICT OVERWRITE WITH"
+    sql parser error: expected (, found: VERSION at line 1, column 56
+    LINE 1: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMNFULL)
+                                                                  ^
 - input: CREATE TABLE T ("FULL" INT) ON CONFLICT DO UPDATE IF NOT NULL
   formatted_sql: CREATE TABLE T ("FULL" INT) ON CONFLICT DO UPDATE IF NOT NULL
 - input: CREATE USER user WITH SUPERUSER CREATEDB PASSWORD 'password'
   formatted_sql: CREATE USER user WITH SUPERUSER CREATEDB PASSWORD 'password'
 - input: CREATE SINK snk
   error_msg: |-
-    sql parser error: Expected FROM or AS after CREATE SINK sink_name, found: EOF at the end
-    Near "CREATE SINK snk"
+    sql parser error: expected FROM or AS after CREATE SINK sink_name, found: EOF at the end
+    LINE 1: CREATE SINK snk
+                           ^
 - input: CREATE SINK IF NOT EXISTS snk FROM mv WITH (connector = 'mysql', mysql.endpoint = '127.0.0.1:3306', mysql.table = '<table_name>', mysql.database = '<database_name>', mysql.user = '<user_name>', mysql.password = '<password>')
   formatted_sql: CREATE SINK IF NOT EXISTS snk FROM mv WITH (connector = 'mysql', mysql.endpoint = '127.0.0.1:3306', mysql.table = '<table_name>', mysql.database = '<database_name>', mysql.user = '<user_name>', mysql.password = '<password>')
 - input: CREATE SINK IF NOT EXISTS snk AS SELECT count(*) AS cnt FROM mv WITH (connector = 'mysql', mysql.endpoint = '127.0.0.1:3306', mysql.table = '<table_name>', mysql.database = '<database_name>', mysql.user = '<user_name>', mysql.password = '<password>')
@@ -90,29 +102,46 @@
   formatted_sql: CREATE SINK snk INTO t AS SELECT * FROM t
 - input: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format;
   error_msg: |-
-    sql parser error: Expected identifier, found: ; at line 1, column 128
-    Near " topic = 'test_topic') format;"
+    sql parser error: expected identifier, found: ; at line 1, column 128
+    LINE 1: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format;
+                                                                                                                                            ^
 - input: create sink sk1 from tt where v1 % 10 = 0 with (connector='blackhole')
   error_msg: |-
-    sql parser error: Expected WITH, found: where at line 1, column 25
-    Near "create sink sk1 from tt"
+    sql parser error: expected WITH, found: where at line 1, column 25
+    LINE 1: create sink sk1 from tt where v1 % 10 = 0 with (connector='blackhole')
+                                   ^
 - input: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format debezium;
   error_msg: |-
-    sql parser error: Expected ENCODE, found: ; at line 1, column 137
-    Near "topic = 'test_topic') format debezium"
+    sql parser error: expected ENCODE, found: ; at line 1, column 137
+    LINE 1: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format debezium;
+                                                                                                                                                    ^
 - input: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format debezium encode;
   error_msg: |-
-    sql parser error: Expected identifier, found: ; at line 1, column 144
-    Near " 'test_topic') format debezium encode;"
+    sql parser error: expected identifier, found: ; at line 1, column 144
+    LINE 1: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format debezium encode;
+                                                                                                                                                            ^
 - input: create user tmp createdb nocreatedb
-  error_msg: 'sql parser error: conflicting or redundant options'
+  error_msg: |-
+    sql parser error: conflicting or redundant options
+    LINE 1: create user tmp createdb nocreatedb
+                                               ^
 - input: create user tmp createdb createdb
-  error_msg: 'sql parser error: conflicting or redundant options'
+  error_msg: |-
+    sql parser error: conflicting or redundant options
+    LINE 1: create user tmp createdb createdb
+                                             ^
 - input: create user tmp with password '123' password null
-  error_msg: 'sql parser error: conflicting or redundant options'
+  error_msg: |-
+    sql parser error: conflicting or redundant options
+    LINE 1: create user tmp with password '123' password null
+                                                             ^
 - input: create user tmp with encrypted password '' password null
-  error_msg: 'sql parser error: conflicting or redundant options'
+  error_msg: |-
+    sql parser error: conflicting or redundant options
+    LINE 1: create user tmp with encrypted password '' password null
+                                                                    ^
 - input: create user tmp with encrypted password null
   error_msg: |-
-    sql parser error: Expected literal string, found: null at line 1, column 41
-    Near " tmp with encrypted password null"
+    sql parser error: expected literal string, found: null at line 1, column 41
+    LINE 1: create user tmp with encrypted password null
+                                                        ^
diff --git a/src/sqlparser/tests/testdata/insert.yaml b/src/sqlparser/tests/testdata/insert.yaml
@@ -1,7 +1,8 @@
 # This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: INSERT public.customer (id, name, active) VALUES (1, 2, 3)
   error_msg: |-
-    sql parser error: Expected INTO, found: public at line 1, column 8
-    Near "INSERT"
+    sql parser error: expected INTO, found: public at line 1, column 8
+    LINE 1: INSERT public.customer (id, name, active) VALUES (1, 2, 3)
+                  ^
 - input: INSERT INTO t VALUES(1,3), (2,4) RETURNING *, a, a as aaa
   formatted_sql: INSERT INTO t VALUES (1, 3), (2, 4) RETURNING (*, a, a AS aaa)