From d1703ccf2e0a823d03112f900ec65a264badc9b6 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 10 Sep 2025 08:41:50 -0700 Subject: [PATCH 01/30] fixes CTE parsing issue and using table as alias --- .../com/clickhouse/jdbc/internal/ClickHouseParser.g4 | 3 ++- .../clickhouse/jdbc/internal/ParsedPreparedStatement.java | 2 +- .../java/com/clickhouse/jdbc/internal/SqlParserTest.java | 8 +++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 index d6b06a28f..e27103f43 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 @@ -55,7 +55,7 @@ columnAliases cteUnboundCol : (literal AS identifier) # CteUnboundColLiteral | (QUERY AS identifier) # CteUnboundColParam - | LPAREN columnExpr RPAREN AS identifier # CteUnboundColExpr + | LPAREN? columnExpr RPAREN? AS identifier # CteUnboundColExpr | LPAREN ctes? selectStmt RPAREN AS identifier # CteUnboundNestedSelect ; @@ -1237,6 +1237,7 @@ keywordForAlias | CURRENT | INDEX | TABLES + | TABLE | TEST | VIEW | PRIMARY diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java index 6fcac9d3f..39d3ab943 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java @@ -136,7 +136,7 @@ public void setHasErrors(boolean hasErrors) { public void enterQueryStmt(ClickHouseParser.QueryStmtContext ctx) { ClickHouseParser.QueryContext qCtx = ctx.query(); if (qCtx != null) { - if (qCtx.selectStmt() != null || qCtx.selectUnionStmt() != null || qCtx.showStmt() != null || qCtx.describeStmt() != null) { + if (qCtx.selectStmt() != null || qCtx.selectUnionStmt() != null || qCtx.showStmt() != null || qCtx.describeStmt() != null || qCtx.ctes() != null) { setHasResultSet(true); } } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 61e84df1b..9321574d2 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -237,6 +237,7 @@ public void testCTEStatements(String sql, int args) { ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertFalse(stmt.isHasErrors()); Assert.assertEquals(stmt.getArgCount(), args); + Assert.assertTrue(stmt.isHasResultSet()); } @DataProvider @@ -248,7 +249,10 @@ public static Object[][] testCTEStmtsDP() { {"with a as (select 1) select * from a; ", 0}, {"(with ? as a select a);", 1}, {"select * from ( with x as ( select 9 ) select * from x );", 0}, - {"WITH toDateTime(?) AS target_time SELECT * FROM table", 1} + {"WITH toDateTime(?) AS target_time SELECT * FROM table", 1}, + {"WITH toDateTime('2025-08-20 12:34:56') AS target_time SELECT * FROM table", 0}, + {"WITH toDate('2025-08-20') as DATE_END, events AS ( SELECT 1 ) SELECT * FROM events", 0}, + {"WITH toDate(?) as DATE_END, events AS ( SELECT 1 ) SELECT * FROM events", 1} }; } @@ -326,6 +330,8 @@ public Object[][] testMiscStmtDp() { {"WITH 'hello' REGEXP 'h' AS result SELECT 1", 0}, {"WITH (select 1) as a, z AS (select 2) SELECT 1", 0}, {"SELECT result FROM test_view(myParam = ?)", 1}, + {"WITH toDate('2025-08-20') as DATE_END, events AS ( SELECT 1 ) SELECT * FROM events", 0}, + {"select 1 table where 1 = ?", 1} }; } From 9f4aa7a47bb86fbaa337126dc9c167f921745b92 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 30 Sep 2025 22:04:53 -0700 Subject: [PATCH 02/30] Added more tests --- .../jdbc/internal/SqlParserTest.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 9321574d2..822c43c8d 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -418,4 +418,67 @@ public Object[][] testMiscStmtDp() { "WHERE\n" + " EventDate = toDate(?) AND\n" + " EventTime <= ts_upper_bound;"; + + + @Test(dataProvider = "testStatementWithoutResultSetDP") + public void testStatementWithoutResultSet(String sql, int args, boolean hasResultSet) { + SqlParser parser = new SqlParser(); + { + ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); + Assert.assertEquals(stmt.getArgCount(), args); + assertEquals(stmt.isHasResultSet(), hasResultSet, "Statement result set expectation does not match"); + Assert.assertFalse(stmt.isHasErrors(), "Statement has errors"); + } + + { + ParsedStatement stmt = parser.parsedStatement(sql); + assertEquals(stmt.isHasResultSet(), hasResultSet, "Statement result set expectation does not match"); + Assert.assertFalse(stmt.isHasErrors(), "Statement has errors"); + } + } + + @DataProvider + public static Object[][] testStatementWithoutResultSetDP() { + return new Object[][]{ + {"INSERT INTO test_table VALUES (1, ?)", 1, false}, + {"SELECT * FROM test_table", 0, true}, + {"CREATE DATABASE `test_db`", 0, false}, + {"CREATE DATABASE `test_db` COMMENT 'for tests'", 0, false}, + {"CREATE DATABASE IF NOT EXISTS `test_db`", 0, false}, + {"CREATE DATABASE IF NOT EXISTS `test_db` ON CLUSTER `cluster`", 0, false}, + {"CREATE DATABASE IF NOT EXISTS `test_db` ON CLUSTER `cluster` ENGINE = Replicated('clickhouse1:9000', 'test_db')", 0, false}, + {"CREATE TABLE `test_table` (id UInt64)", 0, false}, + {"CREATE TABLE IF NOT EXISTS `test_table` (id UInt64)", 0, false}, + {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id", 0, false}, + {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster`", 0, false}, + {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster` ENGINE = Replicated('clickhouse1:9000', 'test_db')", 0, false}, + {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster` ENGINE = Replicated('clickhouse1:9000', 'test_db') COMMENT 'for tests'", 0, false}, + {"CREATE VIEW `test_db`.`source_table` source AS ( SELECT * FROM source_a UNION SELECT * FROM source_b)", 0, false}, + {"CREATE OR REPLACE VIEW `test_db`.`source_table` source AS ( SELECT * FROM source_a UNION SELECT * FROM source_b)", 0, false}, + {"CREATE OR REPLACE VIEW `test_db`.`source_table` source ON CLUSTER `cluster` AS ( SELECT * FROM source_a UNION SELECT * FROM source_b)", 0, false}, + {"CREATE VIEW `test_db`.`source_table` source AS ( SELECT * FROM source_a UNION SELECT * FROM source_b) ENGINE = MaterializedView", 0, false}, + {"CREATE VIEW `test_db`.`source_table` source AS ( SELECT * FROM source_a UNION SELECT * FROM source_b) ENGINE = MaterializedView()", 0, false}, + {"CREATE VIEW `test_db`.`source_table` source AS ( SELECT * FROM source_a UNION SELECT * FROM source_b) ENGINE = MaterializedView() COMMENT 'for tests'", 0, false}, + {"CREATE DICTIONARY `test_db`.dict1 (k1 UInt64 EXPRESSION(k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000)", 0, false}, + {"CREATE DICTIONARY `test_db`.dict1 (k1 UInt64 (k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000) SETTINGS cache_size = 1000 COMMENT 'for tests'", 0, false}, + {"CREATE OR REPLACE DICTIONARY IF NOT EXISTS `test_db`.dict1 (k1 UInt64 (k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000) SETTINGS cache_size = 1000 COMMENT 'for tests'", 0, false}, + {"CREATE OR REPLACE DICTIONARY IF NOT EXISTS `dict1` (k1 UInt64 (k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000) SETTINGS cache_size = 1000 COMMENT 'for tests'", 0, false}, + {"CREATE FUNCTION test_func AS () -> 10", 0, false}, + {"CREATE FUNCTION test_func AS (x) -> 10 * x", 0, false}, + {"CREATE FUNCTION test_func AS (x, y) -> y * x", 0, false}, + {"CREATE FUNCTION test_func ON CLUSTER `cluster` AS (x, y) -> y * x", 0, false}, + {"CREATE USER IF NOT EXISTS `user`", 0, false}, + {"CREATE USER IF NOT EXISTS `user` ON CLUSTER `cluster`", 0, false}, + {"CREATE ROLE IF NOT EXISTS `role1` ON CLUSTER", 0, false}, + {"CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter", 0, false}, + {"CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 TO peter, antonio", 0, false}, + {"CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio", 0, false}, + {"CREATE QUOTA qA FOR INTERVAL 15 month MAX queries = 123 TO CURRENT_USER", 0, false}, + {"CREATE QUOTA qB FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default", 0, false}, + {"CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin", 0, false}, + {"CREATE NAMED COLLECTION foobar AS a = '1', b = '2' OVERRIDABLE", 0, false}, + + + }; + } } \ No newline at end of file From 2113b87ebd238be676ba00c789f87ba089ce45f5 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 3 Oct 2025 15:25:41 -0700 Subject: [PATCH 03/30] filled test statements --- .../jdbc/internal/SqlParserTest.java | 159 +++++++++++++++++- 1 file changed, 155 insertions(+), 4 deletions(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 822c43c8d..119e40aaf 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -331,7 +331,10 @@ public Object[][] testMiscStmtDp() { {"WITH (select 1) as a, z AS (select 2) SELECT 1", 0}, {"SELECT result FROM test_view(myParam = ?)", 1}, {"WITH toDate('2025-08-20') as DATE_END, events AS ( SELECT 1 ) SELECT * FROM events", 0}, - {"select 1 table where 1 = ?", 1} + {"select 1 table where 1 = ?", 1}, + {"insert into t (i, t) values (1, timestamp '2010-01-01 00:00:00')", 0}, + {"insert into t (i, t) values (1, date '2010-01-01')", 0 + } }; } @@ -421,7 +424,7 @@ public Object[][] testMiscStmtDp() { @Test(dataProvider = "testStatementWithoutResultSetDP") - public void testStatementWithoutResultSet(String sql, int args, boolean hasResultSet) { + public void testStatementsForResultSet(String sql, int args, boolean hasResultSet) { SqlParser parser = new SqlParser(); { ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); @@ -440,8 +443,68 @@ public void testStatementWithoutResultSet(String sql, int args, boolean hasResul @DataProvider public static Object[][] testStatementWithoutResultSetDP() { return new Object[][]{ - {"INSERT INTO test_table VALUES (1, ?)", 1, false}, + /* has result set */ {"SELECT * FROM test_table", 0, true}, + {"SHOW CREATE TABLE `db`.`test_table`", 0, true}, + {"SHOW CREATE TEMPORARY TABLE `db1`.`tmp_table`", 0, true}, + {"SHOW CREATE DICTIONARY dict1", 0, true}, + {"SHOW CREATE VIEW view1", 0, true}, + {"SHOW CREATE DATABASE db1", 0, true}, + {"SHOW CREATE TABLE table1 INTO OUTFILE table1.sql", 0, true}, + {"SHOW TABLES ", 0, true}, + {"SHOW TABLES FROM system LIKE '%user%'", 0, true}, + {"SHOW COLUMNS FROM 'orders' LIKE 'delivery_%'", 0, true}, + {"SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2", 0, true}, + {"SHOW INDEX FROM 'tbl'", 0, true}, + {"SHOW PROCESSLIST", 0, true}, + {"SHOW GRANTS FOR `user01`", 0, true}, + {"SHOW GRANTS FOR `user01` FINAL", 0, true}, + {"SHOW GRANTS FOR `user01` WITH IMPLICIT FINAL", 0, true}, + {"SHOW CREATE USER `user01`", 0, true}, + {"SHOW CREATE USER CURRENT_USER", 0, true}, + {"SHOW CREATE ROLE `role_01`", 0, true}, + {"SHOW CREATE POLICY policy_1 ON `tableA`, `db1`.`tableB`", 0, true}, + {"SHOW CREATE ROW POLICY policy_1 ON `tableA`, `db1`.`tableB`", 0, true}, + {"SHOW CREATE QUOTA CURRENT", 0, true}, + {"SHOW CREATE QUOTA `q1`", 0, true}, + {"SHOW CREATE PROFILE `p1`", 0, true}, + {"SHOW CREATE SETTINGS PROFILE `p3`", 0, true}, + {"SHOW USERS", 0, true}, + {"SHOW CURRENT ROLES", 0, true}, + {"SHOW ENABLED ROLES", 0, true}, + {"SHOW SETTINGS PROFILES", 0, true}, + {"SHOW PROFILES", 0, true}, + {"SHOW POLICIES ON `db`.`table`", 0, true}, + {"SHOW ROW POLICIES ON table1", 0, true}, + {"SHOW QUOTAS", 0, true}, + {"SHOW CURRENT QUOTA", 0, true}, + {"SHOW QUOTA", 0, true}, + {"SHOW ACCESS", 0, true}, + {"SHOW CLUSTER `default`", 0, true}, + {"SHOW CLUSTERS LIKE 'test%' LIMIT 1", 0, true}, + {"SHOW SETTINGS LIKE 'send_timeout'", 0, true}, + {"SHOW SETTINGS ILIKE '%CONNECT_timeout%'", 0, true}, + {"SHOW CHANGED SETTINGS ILIKE '%MEMORY%'", 0, true}, + {"SHOW SETTING `min_insert_block_size_rows`", 0, true}, + {"SHOW FILESYSTEM CACHES", 0, true}, + {"SHOW ENGINES", 0, true}, + {"SHOW FUNCTIONS", 0, true}, + {"SHOW FUNCTIONS LIKE '%max%", 0, true}, + {"SHOW MERGES", 0, true}, + {"SHOW MERGES LIKE 'your_t%' LIMIT 1", 0, true}, + {"EXPLAIN SELECT sum(number) FROM numbers(10) GROUP BY number % ?;", 1, true}, + {"EXPLAIN SELECT sum(number) FROM numbers(10) GROUP BY number % 4;", 0, true}, + {"DESCRIBE TABLE table", 0, true}, + {"DESC TABLE table1", 0, true}, + {"EXISTS TABLE `db`.`table01`", 0, true}, + {"EXISTS TABLE ?", 1, true}, + {"CHECK GRANT SELECT(col2) ON table_2", 0, true}, + {"CHECK TABLE test_table", 0, true}, + {"CHECK TABLE t0 PARTITION ID '201003' FORMAT PrettyCompactMonoBlock SETTINGS check_query_single_value_result = 0", 0, true}, + + + /* no result set */ + {"INSERT INTO test_table VALUES (1, ?)", 1, false}, {"CREATE DATABASE `test_db`", 0, false}, {"CREATE DATABASE `test_db` COMMENT 'for tests'", 0, false}, {"CREATE DATABASE IF NOT EXISTS `test_db`", 0, false}, @@ -450,6 +513,8 @@ public static Object[][] testStatementWithoutResultSetDP() { {"CREATE TABLE `test_table` (id UInt64)", 0, false}, {"CREATE TABLE IF NOT EXISTS `test_table` (id UInt64)", 0, false}, {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id", 0, false}, + {"CREATE TABLE `test_table` (id UInt64 NOT NULL ) ENGINE = MergeTree() ORDER BY id", 0, false}, + {"CREATE TABLE `test_table` (id UInt64 NULL ) ENGINE = MergeTree() ORDER BY id", 0, false}, {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster`", 0, false}, {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster` ENGINE = Replicated('clickhouse1:9000', 'test_db')", 0, false}, {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster` ENGINE = Replicated('clickhouse1:9000', 'test_db') COMMENT 'for tests'", 0, false}, @@ -477,7 +542,93 @@ public static Object[][] testStatementWithoutResultSetDP() { {"CREATE QUOTA qB FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default", 0, false}, {"CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin", 0, false}, {"CREATE NAMED COLLECTION foobar AS a = '1', b = '2' OVERRIDABLE", 0, false}, - + {"ALTER TABLE table1 ALTER COLUMN value Int64", 0, false}, + {"alter table t alter column j default 1", 0, false}, + {"alter table t modify comment 'comment'", 0, false}, + + {"DELETE FROM db.table1 ON CLUSTER `default` WHERE max(a, 10) > ?", 1, false}, + {"DELETE FROM table WHERE a = ?", 1, false}, + {"DELETE FROM table WHERE a = ? AND b = ?", 2, false}, + {"DELETE FROM hits WHERE Title LIKE '%hello%';", 0, false}, + + {"SYSTEM START FETCHES", 0, false}, + {"SYSTEM RELOAD DICTIONARIES", 0, false}, + {"SYSTEM RELOAD DICTIONARIES ON CLUSTER `default`", 0, false}, + {"GRANT SELECT ON db.* TO john", 0, false}, + {"GRANT ON CLUSTER `default` SELECT(a, b) ON db1.tableA TO `user` WITH GRANT OPTION WITH REPLACE OPTION", 0, false}, + {"GRANT SELECT db.* TO user01 WITH REPLACE OPTION", 0, false}, + {"GRANT ON CLUSTER role1, role2 TO `user01` WITH ADMIN OPTION WITH REPLACE OPTION", 0, false}, + {"GRANT CURRENT GRANTS TO user01", 0, false}, + {"REVOKE SELECT(a,b) ON db1.tableA FROM `user01", 0, false}, + {"REVOKE SELECT ON db1.* FROM ALL", 0, false}, + {"REVOKE SELECT ON db1.* FROM ALL EXCEPT `admin01`", 0, false}, + {"REVOKE SELECT ON db1.* FROM ALL EXCEPT CURRENT USER", 0, false}, + {"REVOKE ON CLUSTER `default` SELECT ON db1.* FROM ALL EXCEPT CURRENT USER", 0, false}, + {"REVOKE ON CLUSTER `blaster` ADMIN OPTION FOR role1, role3 FROM `user01`", 0, false}, + {"REVOKE ON CLUSTER `blaster` role1, role3 FROM ALL EXCEPT CURRENT USER", 0, false}, + {"REVOKE ON CLUSTER `blaster` role1, role3 FROM ALL EXCEPT `very_nice_user`", 0, false}, + {"UPDATE db.table01 ON CLUSTER `default` SET col1 = ?, col2 = ? WHERE col3 > ?", 3, false}, + {"UPDATE hits SET Title = 'Updated Title' WHERE EventDate = today()", 0, false}, + {"ATTACH TABLE test FROM '01188_attach/test' (s String, n UInt8) ENGINE = File(TSV)", 0, false}, + {"ATTACH TABLE test AS REPLICATED", 0, false}, + {"DETACH TABLE test", 0, false}, + {"ATTACH DICTIONARY IF NOT EXISTS db.dict1 ON CLUSTER `default`", 0, false}, + {"ATTACH DATABASE IF NOT EXISTS db1 ENGINE=MergeTree ON CLUSTER `default`", 0, false}, + {"DROP DATABASE `db1`",0, false}, + {"DROP TABLE `db1`.`table01`", 0, false}, + {"DROP DICTIONARY `dict1`", 0, false}, + {"DROP ROLE IF EXISTS `role01`", 0 , false}, + {"DROP POLICY IF EXISTS `pol1`", 0, false}, + {"DROP QUOTA IF EXISTS q1", 0, false}, + {"DROP SETTINGS PROFILE IF EXISTS `profile1` ON CLUSTER `default`", 0, false}, + {"DROP VIEW view1 ON CLUSTER `default` SYNC", 0, false}, + {"DROP FUNCTION linear_equation", 0, false}, + {"DROP NAMED COLLECTION foobar", 0, false}, + {"KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90'", 0, false}, + {"KILL QUERY WHERE user='username' SYNC", 0, false}, + {"KILL QUERY ON CLUSTER `default` WHERE user='username' SYNC", 0, false}, + {"KILL QUERY ON CLUSTER `default` WHERE user='username' ASYNC", 0, false}, + {"KILL QUERY ON CLUSTER `default` WHERE user='username' TEST", 0, false}, + {"KILL MUTATION WHERE database = 'default' AND table = 'table'", 0, false}, + {"KILL MUTATION WHERE database = 'default' AND table = 'table' AND mutation_id = 'mutation_3.txt'", 0, false}, + {"OPTIMIZE TABLE table DEDUPLICATE BY colX,colY,colZ", 0, false}, + {"OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT colX", 0, false}, + {"OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT (colX, colY)", 0, false}, + {"OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex')", 0, false}, + {"OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT colX", 0, false}, + {"OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT (colX, colY)", 0, false}, + {"OPTIMIZE TABLE table DEDUPLICATE", 0, false}, + {"OPTIMIZE TABLE table DEDUPLICATE BY *", 0, false}, + {"RENAME TABLE table_A TO table_A_bak, table_B TO table_B_bak", 0, false}, + {"RENAME TABLE table_A TO table_A_bak, table_B TO table_B_bak ON CLUSTER `default`", 0, false}, + {"RENAME DICTIONARY dictA TO dictB ON CLUSTER `default`", 0, false}, + {"EXCHANGE TABLES table1 AND table2", 0, false}, + {"EXCHANGE TABLES table1 AND table2 ON CLUSTER `default`", 0, false}, + {"EXCHANGE DICTIONARIES dict1 AND dict2", 0, false}, + {"EXCHANGE DICTIONARIES dict1 AND dict2 ON CLUSTER `default`", 0, false}, + {"SET profile = 'profile-name-from-the-settings-file'", 0, false}, + {"SET ROLE role1", 0, false}, + {"SET DEFAULT ROLE role1 TO user", 0, false}, + {"SET DEFAULT ROLE NONE TO user", 0, false}, + {"SET DEFAULT ROLE ALL EXCEPT role1, role2 TO user", 0, false}, + {"TRUNCATE TABLE IF EXISTS `db1`.`table1` ON CLUSTER `default` SYNC", 0, false}, + {"TRUNCATE TABLE `db1`.`table1` ON CLUSTER `default` SYNC", 0, false}, + {"TRUNCATE TABLE `db1`.`table1` ON CLUSTER `default`", 0, false}, + {"TRUNCATE TABLE `db1`.`table1`", 0, false}, + {"TRUNCATE DATABASE IF EXISTS db ON CLUSTER `cluster`", 0, false}, + {"TRUNCATE DATABASE IF EXISTS db", 0, false}, + {"TRUNCATE DATABASE `db`", 0, false}, + {"TRUNCATE ALL TABLES FROM IF EXISTS `db` NOT LIKE 'tmp%' ON CLUSTER `cluster`", 0, false}, + {"TRUNCATE ALL TABLES FROM IF EXISTS `db` NOT LIKE 'tmp%'", 0, false}, + {"TRUNCATE ALL TABLES FROM `db` NOT LIKE 'tmp%' ON CLUSTER `cluster`", 0, false}, + {"TRUNCATE TABLES FROM `db` LIKE 'tmp%' ON CLUSTER `cluster`", 0, false}, + {"TRUNCATE TABLES FROM `db` LIKE 'tmp%'", 0, false}, + {"USE test_db", 0, false}, + {"MOVE USER test TO local_directory", 0, false}, + {"MOVE ROLE test TO memory", 0, false}, + {"UNDROP TABLE tab", 0, false}, + {"UNDROP TABLE db.tab ON CLUSTER `default`", 0, false}, + {"UNDROP TABLE db.tab UUID '857d-4a57-9ee0-327da5d60a90' ON CLUSTER `default`", 0, false}, }; } From 3a96804dc4b30bf8b2b1e4fb1d470f7fe10f0cae Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 3 Oct 2025 15:36:45 -0700 Subject: [PATCH 04/30] moved parser to separate package to track coverage --- .../jdbc/internal/{ => parser}/ClickHouseLexer.g4 | 0 .../jdbc/internal/{ => parser}/ClickHouseParser.g4 | 0 .../clickhouse/jdbc/internal/ParsedPreparedStatement.java | 2 ++ .../java/com/clickhouse/jdbc/internal/ParsedStatement.java | 2 ++ .../main/java/com/clickhouse/jdbc/internal/SqlParser.java | 6 +++--- 5 files changed, 7 insertions(+), 3 deletions(-) rename jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/{ => parser}/ClickHouseLexer.g4 (100%) rename jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/{ => parser}/ClickHouseParser.g4 (100%) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 similarity index 100% rename from jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseLexer.g4 rename to jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 similarity index 100% rename from jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 rename to jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java index 39d3ab943..da5534341 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java @@ -1,6 +1,8 @@ package com.clickhouse.jdbc.internal; import com.clickhouse.client.api.sql.SQLUtils; +import com.clickhouse.jdbc.internal.parser.ClickHouseParser; +import com.clickhouse.jdbc.internal.parser.ClickHouseParserBaseListener; import org.antlr.v4.runtime.tree.ErrorNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java index ee94eaf67..425af8e84 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java @@ -1,6 +1,8 @@ package com.clickhouse.jdbc.internal; import com.clickhouse.client.api.sql.SQLUtils; +import com.clickhouse.jdbc.internal.parser.ClickHouseParser; +import com.clickhouse.jdbc.internal.parser.ClickHouseParserBaseListener; import org.antlr.v4.runtime.tree.ErrorNode; import java.util.ArrayList; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java index 8a57a550d..6812ffc66 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java @@ -1,5 +1,8 @@ package com.clickhouse.jdbc.internal; +import com.clickhouse.jdbc.internal.parser.ClickHouseLexer; +import com.clickhouse.jdbc.internal.parser.ClickHouseParser; +import com.clickhouse.jdbc.internal.parser.ClickHouseParserBaseListener; import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; @@ -10,9 +13,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - public class SqlParser { private static final Logger LOG = LoggerFactory.getLogger(SqlParser.class); From 25cca50139e3caa399d72605dccd0079d02a3c13 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 6 Oct 2025 13:57:06 -0700 Subject: [PATCH 05/30] fixed show statement tests --- .../jdbc/internal/parser/ClickHouseLexer.g4 | 189 +++++++++++++----- .../jdbc/internal/parser/ClickHouseParser.g4 | 42 +++- .../jdbc/internal/SqlParserTest.java | 8 +- 3 files changed, 175 insertions(+), 64 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 index 743e73c1a..10447a97d 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 @@ -9,36 +9,58 @@ lexer grammar ClickHouseLexer; // Keywords + + +ACCESS : A C C E S S; ADD : A D D; +ADMIN : A D M I N; AFTER : A F T E R; ALIAS : A L I A S; ALL : A L L; +ALLOW : A L L O W; ALTER : A L T E R; AND : A N D; ANTI : A N T I; ANY : A N Y; +ARBITRARY : A R B I T R A R Y ; ARRAY : A R R A Y; AS : A S; ASCENDING : A S C | A S C E N D I N G; ASOF : A S O F; AST : A S T; ASYNC : A S Y N C; +ASYNCHRONOUS : A S Y N C H R O N O U S ; ATTACH : A T T A C H; +AZURE : A Z U R E; +BACKUP : B A C K U P; +BCRYPT_HASH : B C R Y P T '_' H A S H; +BCRYPT_PASSWORD : B C R Y P T '_' P A S S W O R D; BETWEEN : B E T W E E N; +BLOCKING : B L O C K I N G ; BOTH : B O T H; BY : B Y; -BCRYPT_PASSWORD : B C R Y P T '_' P A S S W O R D; -BCRYPT_HASH : B C R Y P T '_' H A S H; +CACHE : C A C H E ; +CACHES : C A C H E S ; CASE : C A S E; CAST : C A S T; +CHANGED : C H A N G E D; CHECK : C H E C K; +CLEANUP : C L E A N U P; CLEAR : C L E A R; +CLIENT : C L I E N T ; CLUSTER : C L U S T E R; +CLUSTERS : C L U S T E R S; CN : C N; CODEC : C O D E C; COLLATE : C O L L A T E; +COLLECTION : C O L L E C T I O N ; +COLLECTIONS : C O L L E C T I O N S ; COLUMN : C O L U M N; +COLUMNS : C O L U M N S ; COMMENT : C O M M E N T; +COMPILED : C O M P I L E D ; +CONFIG : C O N F I G ; +CONNECTIONS : C O N N E C T I O N S ; CONSTRAINT : C O N S T R A I N T; CREATE : C R E A T E; CROSS : C R O S S; @@ -51,6 +73,7 @@ DATE : D A T E; DAY : D A Y; DEDUPLICATE : D E D U P L I C A T E; DEFAULT : D E F A U L T; +DEFINER : D E F I N E R; DELAY : D E L A Y; DELETE : D E L E T E; DESC : D E S C; @@ -62,19 +85,28 @@ DICTIONARY : D I C T I O N A R Y; DISK : D I S K; DISTINCT : D I S T I N C T; DISTRIBUTED : D I S T R I B U T E D; -DOUBLE_SHA1_PASSWORD : D O U B L E '_' S H A '1' '_' P A S S W O R D; +DNS : D N S ; DOUBLE_SHA1_HASH : D O U B L E '_' S H A '1' '_' H A S H; +DOUBLE_SHA1_PASSWORD : D O U B L E '_' S H A '1' '_' P A S S W O R D; DROP : D R O P; ELSE : E L S E; +EMBEDDED : E M B E D D E D ; +ENABLED : E N A B L E D; END : E N D; ENGINE : E N G I N E; +ENGINES : E N G I N E S; EVENTS : E V E N T S; +EXCEPT : E X C E P T; EXISTS : E X I S T S; EXPLAIN : E X P L A I N; EXPRESSION : E X P R E S S I O N; -EXCEPT : E X C E P T; +EXTENDED : E X T E N D E D; EXTRACT : E X T R A C T; +FAILPOINT : F A I L P O I N T ; FETCHES : F E T C H E S; +FETCH : F E T C H ; +FILE : F I L E; +FILESYSTEM : F I L E S Y S T E M ; FINAL : F I N A L; FIRST : F I R S T; FLUSH : F L U S H; @@ -85,33 +117,47 @@ FREEZE : F R E E Z E; FROM : F R O M; FULL : F U L L; FUNCTION : F U N C T I O N; +FUNCTIONS : F U N C T I O N S; +FUZZER : F U Z Z E R ; GLOBAL : G L O B A L; -GRANULARITY : G R A N U L A R I T Y; GRANTEES : G R A N T E E S; +GRANT : G R A N T; +GRANTS : G R A N T S; +GRANULARITY : G R A N U L A R I T Y; GROUP : G R O U P; HAVING : H A V I N G; +HDFS : H D F S; HIERARCHICAL : H I E R A R C H I C A L; -HTTP : H T T P; +HIVE : H I V E; HOST : H O S T; HOUR : H O U R; -ID : I D; +HTTP : H T T P; IDENTIFIED : I D E N T I F I E D; +ID : I D; IF : I F; ILIKE : I L I K E; -IN : I N; +IMPLICIT : I M P L I C I T; +INDEXES : I N D E X E S; INDEX : I N D E X; +INDICES : I N D I C E S; INF : I N F | I N F I N I T Y; +IN : I N; INJECTIVE : I N J E C T I V E; INNER : I N N E R; INSERT : I N S E R T; INTERVAL : I N T E R V A L; INTO : I N T O; +INTROSPECTION : I N T R O S P E C T I O N; IP : I P; IS : I S; IS_OBJECT_ID : I S UNDERSCORE O B J E C T UNDERSCORE I D; +JDBC : J D B C; +JEMALLOC : J E M A L L O C ; JOIN : J O I N; -KEY : K E Y; +KAFKA : K A F K A; KERBEROS : K E R B E R O S; +KEY : K E Y; +KEYS : K E Y S; KILL : K I L L; LAST : L A S T; LAYOUT : L A Y O U T; @@ -121,144 +167,179 @@ LEFT : L E F T; LIFETIME : L I F E T I M E; LIKE : L I K E; LIMIT : L I M I T; +LISTEN : L I S T E N ; LIVE : L I V E; +LOADING : L O A D I N G; +LOAD : L O A D ; LOCAL : L O C A L; +LOG : L O G ; LOGS : L O G S; -MATERIALIZE : M A T E R I A L I Z E; +MANAGEMENT : M A N A G E M E N T; +MARK : M A R K ; MATERIALIZED : M A T E R I A L I Z E D; +MATERIALIZE : M A T E R I A L I Z E; MAX : M A X; MERGES : M E R G E S; +METRICS : M E T R I C S ; MIN : M I N; MINUTE : M I N U T E; +MMAP : M M A P ; +MODEL : M O D E L ; MODIFY : M O D I F Y; +MONGO : M O N G O; MONTH : M O N T H; MOVE : M O V E; +MOVES : M O V E S ; MUTATION : M U T A T I O N; -NAN_SQL : N A N; // conflicts with macro NAN +MYSQL : M Y S Q L; +NAMED : N A M E D ; NAME : N A M E; +NAN_SQL : N A N; // conflicts with macro NAN +NATS : N A T S; +NONE : N O N E; NO : N O; NO_PASSWORD : N O '_' P A S S W O R D; -NONE : N O N E; NOT : N O T; -NULL_SQL : N U L L; // conflicts with macro NULL NULLS : N U L L S; +NULL_SQL : N U L L; // conflicts with macro NULL +ODBC : O D B C; OFFSET : O F F S E T; ON : O N; OPTIMIZE : O P T I M I Z E; -OR : O R; +OPTION : O P T I O N; ORDER : O R D E R; +OR : O R; OUTER : O U T E R; OUTFILE : O U T F I L E; OVER : O V E R; +PAGE : P A G E ; PARTITION : P A R T I T I O N; +PARTS : P A R T S ; +PERMISSIVE : P E R M I S S I V E; +PLAINTEXT_PASSWORD : P L A I N T E X T '_' P A S S W O R D; +POLICIES : P O L I C I E S ; +POLICY : P O L I C Y; POPULATE : P O P U L A T E; +POSTGRES : P O S T G R E S; PRECEDING : P R E C E D I N G; PREWHERE : P R E W H E R E; PRIMARY : P R I M A R Y; +PROCESSLIST : P R O C E S S L I S T; +PROFILE : P R O F I L E; +PROFILES : P R O F I L E S; PROJECTION : P R O J E C T I O N; -PLAINTEXT_PASSWORD : P L A I N T E X T '_' P A S S W O R D; +PULLING : P U L L I N G ; QUARTER : Q U A R T E R; +QUEUE : Q U E U E ; +QUEUES : Q U E U E S ; +QUOTA : Q U O T A; +QUOTAS : Q U O T A S ; +RABBITMQ : R A B B I T M Q; RANGE : R A N G E; +READINESS : R E A D I N E S S ; REALM : R E A L M; +REDIS : R E D I S; +REDUCE : R E D U C E ; +REFRESH : R E F R E S H ; REGEXP : R E G E X P; RELOAD : R E L O A D; +REMOTE : R E M O T E; REMOVE : R E M O V E; RENAME : R E N A M E; REPLACE : R E P L A C E; REPLICA : R E P L I C A; REPLICATED : R E P L I C A T E D; +REPLICATION : R E P L I C A T I O N ; +RESOURCE : R E S O U R C E ; +RESTART : R E S T A R T; +RESTORE : R E S T O R E ; +RESTRICTIVE : R E S T R I C T I V E; RIGHT : R I G H T; ROLE : R O L E; +ROLES : R O L E S ; ROLLUP : R O L L U P; ROW : R O W; ROWS : R O W S; +S3 : S '3'; SAMPLE : S A M P L E; SCHEMA : S C H E M A; -SCRAM_SHA256_PASSWORD : S C R A M '_' S H A '2' '5' '6' '_' P A S S W O R D; SCRAM_SHA256_HASH : S C R A M '_' S H A '2' '5' '6' '_' H A S H; +SCRAM_SHA256_PASSWORD : S C R A M '_' S H A '2' '5' '6' '_' P A S S W O R D; SECOND : S E C O N D; +SECRETS : S E C R E T S ; +SECURITY : S E C U R I T Y; SELECT : S E L E C T; SEMI : S E M I; SENDS : S E N D S; SERVER : S E R V E R; -SSL_CERTIFICATE : S S L '_' C E R T I F I C A T E; -SSH_KEY : S S H '_' K E Y; SET : S E T; +SETTING : S E T T I N G; SETTINGS : S E T T I N G S; -SHOW : S H O W; -SHA256_PASSWORD : S H A '2' '5' '6' '_' P A S S W O R D; SHA256_HASH : S H A '2' '5' '6' '_' H A S H; +SHA256_PASSWORD : S H A '2' '5' '6' '_' P A S S W O R D; +SHARDS : S H A R D S; +SHOW : S H O W; +SHUTDOWN : S H U T D O W N ; SOURCE : S O U R C E; +SOURCES : S O U R C E S; +SQLITE : S Q L I T E; +SQL : S Q L; +SSH_KEY : S S H '_' K E Y; +SSL_CERTIFICATE : S S L '_' C E R T I F I C A T E; START : S T A R T; +STATISTICS : S T A T I S T I C S ; STOP : S T O P; SUBSTRING : S U B S T R I N G; SYNC : S Y N C; SYNTAX : S Y N T A X; SYSTEM : S Y S T E M; -TABLE : T A B L E; TABLES : T A B L E S; +TABLE : T A B L E; TEMPORARY : T E M P O R A R Y; TEST : T E S T; THEN : T H E N; +THREAD : T H R E A D ; TIES : T I E S; TIMEOUT : T I M E O U T; TIMESTAMP : T I M E S T A M P; -TO : T O; TOP : T O P; TOTALS : T O T A L S; +TO : T O; TRAILING : T R A I L I N G; +TRANSACTION : T R A N S A C T I O N; TRIM : T R I M; TRUNCATE : T R U N C A T E; TTL : T T L; TYPE : T Y P E; UNBOUNDED : U N B O U N D E D; +UNCOMPRESSED : U N C O M P R E S S E D ; +UNDROP : U N D R O P; +UNFREEZE : U N F R E E Z E ; UNION : U N I O N; +UNLOAD : U N L O A D ; +UNTIL : U N T I L; UPDATE : U P D A T E; -USE : U S E; +URL : U R L; +USERS : U S E R S ; USER : U S E R; +USE : U S E; USING : U S I N G; UUID : U U I D; +VALID : V A L I D; VALUES : V A L U E S; +VIEWS : V I E W S; VIEW : V I E W; +VIRTUAL : V I R T U A L; VOLUME : V O L U M E; +WAIT : W A I T; WATCH : W A T C H; WEEK : W E E K; WHEN : W H E N; WHERE : W H E R E; WINDOW : W I N D O W; WITH : W I T H; +WORKLOAD : W O R K L O A D ; YEAR : Y E A R | Y Y Y Y; -QUOTA : Q U O T A; -ACCESS : A C C E S S; -GRANT : G R A N T; -WAIT : W A I T; -CLEANUP : C L E A N U P; -DEFINER : D E F I N E R; -RESTART : R E S T A R T; -SOURCES : S O U R C E S; -AZURE : A Z U R E; -FILE : F I L E; -HDFS : H D F S; -HIVE : H I V E; -JDBC : J D B C; -KAFKA : K A F K A; -MONGO : M O N G O; -MYSQL : M Y S Q L; -NATS : N A T S; -ODBC : O D B C; -POSTGRES : P O S T G R E S; -RABBITMQ : R A B B I T M Q; -REDIS : R E D I S; -REMOTE : R E M O T E; -S3 : S '3'; -SQLITE : S Q L I T E; -URL : U R L; -LOADING : L O A D I N G; -VIRTUAL : V I R T U A L; -VIEWS : V I E W S; -POLICY : P O L I C Y; -PERMISSIVE : P E R M I S S I V E; -RESTRICTIVE : R E S T R I C T I V E; JSON_FALSE : 'false'; JSON_TRUE : 'true'; diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index e27103f43..6a6de4cb1 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -830,12 +830,34 @@ systemPrivilege // SHOW statements showStmt - : SHOW CREATE DATABASE databaseIdentifier # showCreateDatabaseStmt - | SHOW CREATE DICTIONARY tableIdentifier # showCreateDictionaryStmt - | SHOW CREATE TEMPORARY? TABLE? tableIdentifier # showCreateTableStmt - | SHOW DATABASES # showDatabasesStmt - | SHOW DICTIONARIES (FROM databaseIdentifier)? # showDictionariesStmt - | SHOW TEMPORARY? TABLES ((FROM | IN) databaseIdentifier)? (LIKE STRING_LITERAL | whereClause)? limitClause? # showTablesStmt + : SHOW CREATE? (TEMPORARY? TABLE | DICTIONARY | VIEW | DATABASE) tableIdentifier (INTO OUTFILE literal)? (FORMAT identifier) # showCreateStmt + | SHOW DATABASES (NOT? (LIKE | ILIKE) literal) (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showDatabasesStmt + | SHOW FULL? TEMPORARY? TABLES ((FROM | IN) identifier)? (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showTablesStmt + | SHOW EXTENDED? FULL? COLUMNS ((FROM | IN) identifier (FROM | IN) identifier)? (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showColumnsStmt + | SHOW DICTIONARIES ((FROM | IN) identifier)? (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showDictionariesStmt + | SHOW EXTENDED? (INDEX | INDEXES | INDICES | KEYS ) (FROM | IN) identifier ((FROM | IN) identifier)? (WHERE columnExpr) (INTO OUTFILE filename)? (FORMAT identifier)? # showIndexStmt + | SHOW PROCESSLIST (INTO OUTFILE filename)? (FORMAT identifier)? # showProcessListStmt + | SHOW GRANTS (FOR identifier (COMMA identifier)*)? (WITH IMPLICIT)? FINAL? # showGrantsStmt + | SHOW CREATE USER ((identifier (COMMA identifier)*) | CURRENT_USER) # showCreateUserStmt + | SHOW CREATE ROLE (identifier (COMMA identifier)*) # showCreateRoleStmt + | SHOW CREATE ROW? POLICY identifier ON tableIdentifier # showCreatePolicyStmt + | SHOW CREATE QUOTA ((identifier (COMMA identifier)*) | CURRENT) # showCreateQuotaStmt + | SHOW CREATE (SETTINGS)? PROFILE identifier (COMMA identifier)* # showCreateProfile + | SHOW USERS # showUsersStmt + | SHOW (CURRENT|ENABLED)? ROLES # showRolesStmt + | SHOW SETTINGS? PROFILES # showProfilesStmt + | SHOW ROW? POLICIES (ON identifier)? # showPoliciesStmt + | SHOW QUOTAS # showQuotasStmt + | SHOW CURRENT? QUOTA # showQuotaStmt + | SHOW ACCESS # showAccessStmt + | SHOW CLUSTER identifier # showClusterStmt + | SHOW CLUSTERS (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showClustersStmt + | SHOW CHANGED? SETTINGS (LIKE | ILIKE) literal # showSettingsStmt + | SHOW SETTING identifier # showSettingStmt + | SHOW FILESYSTEM CACHES # showFSCachesStmt + | SHOW ENGINES (INTO OUTFILE filename)? (FORMAT identifier)? # showEnginesStmt + | SHOW FUNCTIONS (NOT? (LIKE | ILIKE) literal)? # showFunctionsStmt + | SHOW MERGES (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showMergesStmt ; // SYSTEM statements @@ -986,6 +1008,10 @@ tableIdentifier : (databaseIdentifier DOT)? identifier ; +viewIdentifier + : tableIdentifier + ; + tableArgList : tableArgExpr (COMMA tableArgExpr)* ; @@ -1027,6 +1053,10 @@ literal | NULL_SQL ; +filename + : STRING_LITERAL + ; + interval : SECOND | MINUTE diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 119e40aaf..7cc5863ff 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -450,12 +450,12 @@ public static Object[][] testStatementWithoutResultSetDP() { {"SHOW CREATE DICTIONARY dict1", 0, true}, {"SHOW CREATE VIEW view1", 0, true}, {"SHOW CREATE DATABASE db1", 0, true}, - {"SHOW CREATE TABLE table1 INTO OUTFILE table1.sql", 0, true}, + {"SHOW CREATE TABLE table1 INTO OUTFILE 'table1.sql'", 0, true}, {"SHOW TABLES ", 0, true}, {"SHOW TABLES FROM system LIKE '%user%'", 0, true}, - {"SHOW COLUMNS FROM 'orders' LIKE 'delivery_%'", 0, true}, + {"SHOW COLUMNS FROM `orders` LIKE 'delivery_%'", 0, true}, {"SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2", 0, true}, - {"SHOW INDEX FROM 'tbl'", 0, true}, + {"SHOW INDEX FROM `tbl`", 0, true}, {"SHOW PROCESSLIST", 0, true}, {"SHOW GRANTS FOR `user01`", 0, true}, {"SHOW GRANTS FOR `user01` FINAL", 0, true}, @@ -489,7 +489,7 @@ public static Object[][] testStatementWithoutResultSetDP() { {"SHOW FILESYSTEM CACHES", 0, true}, {"SHOW ENGINES", 0, true}, {"SHOW FUNCTIONS", 0, true}, - {"SHOW FUNCTIONS LIKE '%max%", 0, true}, + {"SHOW FUNCTIONS LIKE '%max%'", 0, true}, {"SHOW MERGES", 0, true}, {"SHOW MERGES LIKE 'your_t%' LIMIT 1", 0, true}, {"EXPLAIN SELECT sum(number) FROM numbers(10) GROUP BY number % ?;", 1, true}, From 6f7e4150841bdf23bbc0cbcf6ecef89ecf449b1b Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 6 Oct 2025 23:23:12 -0700 Subject: [PATCH 06/30] Fixed statements with result set and several other --- .../jdbc/internal/parser/ClickHouseLexer.g4 | 17 ++++++-- .../jdbc/internal/parser/ClickHouseParser.g4 | 39 ++++++++++++------- .../internal/ParsedPreparedStatement.java | 7 +--- .../jdbc/internal/ParsedStatement.java | 8 +--- .../clickhouse/jdbc/internal/SqlParser.java | 8 ++++ .../jdbc/internal/SqlParserTest.java | 25 ++++++------ 6 files changed, 64 insertions(+), 40 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 index 10447a97d..444460994 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 @@ -45,6 +45,7 @@ CASE : C A S E; CAST : C A S T; CHANGED : C H A N G E D; CHECK : C H E C K; +CHANGEABLE_IN_READONLY : C H A N G E A B L E UNDERSCORE I N UNDERSCORE R E A D O N L Y; CLEANUP : C L E A N U P; CLEAR : C L E A R; CLIENT : C L I E N T ; @@ -62,6 +63,7 @@ COMPILED : C O M P I L E D ; CONFIG : C O N F I G ; CONNECTIONS : C O N N E C T I O N S ; CONSTRAINT : C O N S T R A I N T; +CONST : C O N S T; CREATE : C R E A T E; CROSS : C R O S S; CUBE : C U B E; @@ -95,6 +97,7 @@ ENABLED : E N A B L E D; END : E N D; ENGINE : E N G I N E; ENGINES : E N G I N E S; +ESTIMATE : E S T I M A T E; EVENTS : E V E N T S; EXCEPT : E X C E P T; EXISTS : E X I S T S; @@ -142,6 +145,7 @@ INDEX : I N D E X; INDICES : I N D I C E S; INF : I N F | I N F I N I T Y; IN : I N; +INHERIT : I N H E R I T; INJECTIVE : I N J E C T I V E; INNER : I N N E R; INSERT : I N S E R T; @@ -212,11 +216,15 @@ OR : O R; OUTER : O U T E R; OUTFILE : O U T F I L E; OVER : O V E R; +OVERRIDE : O V E R R I D E; PAGE : P A G E ; PARTITION : P A R T I T I O N; -PARTS : P A R T S ; +PART : P A R T; +PARTS : P A R T S; PERMISSIVE : P E R M I S S I V E; +PIPELINE : P I P E L I N E; PLAINTEXT_PASSWORD : P L A I N T E X T '_' P A S S W O R D; +PLAN : P L A N; POLICIES : P O L I C I E S ; POLICY : P O L I C Y; POPULATE : P O P U L A T E; @@ -236,8 +244,9 @@ QUOTA : Q U O T A; QUOTAS : Q U O T A S ; RABBITMQ : R A B B I T M Q; RANGE : R A N G E; -READINESS : R E A D I N E S S ; +READINESS : R E A D I N E S S; REALM : R E A L M; +READONLY : R E A D O N L Y; REDIS : R E D I S; REDUCE : R E D U C E ; REFRESH : R E F R E S H ; @@ -307,6 +316,7 @@ TOTALS : T O T A L S; TO : T O; TRAILING : T R A I L I N G; TRANSACTION : T R A N S A C T I O N; +TREE : T R E E; TRIM : T R I M; TRUNCATE : T R U N C A T E; TTL : T T L; @@ -338,7 +348,8 @@ WHEN : W H E N; WHERE : W H E R E; WINDOW : W I N D O W; WITH : W I T H; -WORKLOAD : W O R K L O A D ; +WORKLOAD : W O R K L O A D; +WRITABLE : W R I T A B L E; YEAR : Y E A R | Y Y Y Y; JSON_FALSE : 'false'; diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index 6a6de4cb1..bfb893a09 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -89,6 +89,7 @@ alterTableClause | MODIFY COLUMN (IF EXISTS)? tableColumnDfnt # AlterTableClauseModify | MODIFY ORDER BY columnExpr # AlterTableClauseModifyOrderBy | MODIFY ttlClause # AlterTableClauseModifyTTL + | MODIFY COMMENT literal # AlterTableClauseModifyComment | MOVE partitionClause ( TO DISK STRING_LITERAL | TO VOLUME STRING_LITERAL @@ -130,7 +131,9 @@ attachStmt // CHECK statement checkStmt - : CHECK TABLE tableIdentifier partitionClause? + : CHECK TABLE tableIdentifier (PARTITION identifier | PART identifier)? (FORMAT identifier)? settingsClause? # checkTableStmt + | CHECK ALL TABLES (FORMAT identifier)? settingsClause? # checkAllTablesStmt + | CHECK GRANT privilege columnsClause? ON grantTableIdentifier # checkGrantStmt ; // CREATE statement @@ -138,7 +141,7 @@ checkStmt createStmt : (ATTACH | CREATE) DATABASE (IF NOT EXISTS)? databaseIdentifier clusterClause? engineExpr? # CreateDatabaseStmt | (ATTACH | CREATE (OR REPLACE)? | REPLACE) DICTIONARY (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? dictionarySchemaClause - dictionaryEngineClause # CreateDictionaryStmt + dictionaryEngineClause sourceClause layoutClause lifetimeClause dictionarySettingsClause? (COMMENT literal)? # CreateDictionaryStmt | (ATTACH | CREATE) LIVE VIEW (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? ( WITH TIMEOUT DECIMAL_LITERAL? )? destinationClause? tableSchemaClause? subqueryClause # CreateLiveViewStmt @@ -147,8 +150,8 @@ createStmt | engineClause POPULATE? ) subqueryClause # CreateMaterializedViewStmt | (ATTACH | CREATE (OR REPLACE)? | REPLACE) TEMPORARY? TABLE (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? tableSchemaClause? - engineClause? subqueryClause? # CreateTableStmt - | (ATTACH | CREATE) (OR REPLACE)? VIEW (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? tableSchemaClause? subqueryClause # + engineClause? subqueryClause? # CreateTableStmt + | (ATTACH | CREATE) (OR REPLACE)? VIEW (IF NOT EXISTS)? tableIdentifier alias? uuidClause? clusterClause? tableSchemaClause? subqueryClause # CreateViewStmt | CREATE USER ((IF NOT EXISTS) | (OR REPLACE))? userIdentifier (COMMA userIdentifier)* clusterClause? userIdentifiedClause? @@ -162,6 +165,16 @@ createStmt | CREATE (ROW)? POLICY (IF NOT EXISTS | OR REPLACE)? identifier clusterClause? ON tableIdentifier (IN identifier)? (AS (PERMISSIVE | RESTRICTIVE))? (FOR SELECT)? USING columnExpr (TO identifier | ALL | ALL EXCEPT identifier)? # CreatePolicyStmt + | CREATE SETTINGS? PROFILE ((IF NOT EXISTS) | (OR REPLACE))? identifier (COMMA identifier)* clusterClause? + (IN identifier)? ((SETTINGS identifier (EQ_SINGLE literal)? (MIN EQ_SINGLE? literal)? (MAX EQ_SINGLE? literal)? + (CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY)?) + | ( INHERIT identifier))? (TO identifier | ALL | ALL EXCEPT identifier)? # createProfileStmt + | CREATE FUNCTION identifier clusterClause? AS LPAREN (identifier)? (COMMA identifier)? RPAREN ARROW .+? #createFunctionStmt + | CREATE NAMED COLLECTION (IF NOT EXISTS)? identifier clusterClause? AS nameCollectionKey (COMMA nameCollectionKey)* #createNamedCollectionStmt + ; + +nameCollectionKey + : (identifier EQ_SINGLE literal (NOT? OVERRIDE)?) ; userIdentifier @@ -205,7 +218,7 @@ dictionarySchemaClause ; dictionaryAttrDfnt - : identifier columnTypeExpr + : identifier columnTypeExpr ((DEFAULT | EXPRESSION) columnExpr)? (IS_OBJECT_ID|HIERARCHICAL|INJECTIVE)? ; dictionaryEngineClause @@ -213,7 +226,7 @@ dictionaryEngineClause ; dictionaryPrimaryKeyClause - : PRIMARY KEY columnExprList + : PRIMARY KEY (identifier) (COMMA identifier)* ; dictionaryArgExpr @@ -221,7 +234,7 @@ dictionaryArgExpr ; sourceClause - : SOURCE LPAREN identifier LPAREN dictionaryArgExpr* RPAREN RPAREN + : SOURCE LPAREN identifier LPAREN settingExprList RPAREN RPAREN ; lifetimeClause @@ -298,7 +311,7 @@ tableElementExpr ; tableColumnDfnt - : nestedIdentifier columnTypeExpr tableColumnPropertyExpr? (COMMENT STRING_LITERAL)? codecExpr? ( + : nestedIdentifier columnTypeExpr (NULL_SQL | NOT NULL_SQL)? tableColumnPropertyExpr? (COMMENT STRING_LITERAL)? codecExpr? ( TTL columnExpr )? | nestedIdentifier columnTypeExpr? tableColumnPropertyExpr (COMMENT STRING_LITERAL)? codecExpr? ( @@ -307,7 +320,7 @@ tableColumnDfnt ; tableColumnPropertyExpr - : (DEFAULT | MATERIALIZED | ALIAS) columnExpr + : (DEFAULT | MATERIALIZED | ALIAS ) columnExpr ; tableIndexDfnt @@ -348,15 +361,15 @@ dropStmt // EXISTS statement existsStmt - : EXISTS DATABASE databaseIdentifier # ExistsDatabaseStmt - | EXISTS (DICTIONARY | TEMPORARY? TABLE | VIEW)? tableIdentifier # ExistsTableStmt + : EXISTS DATABASE databaseIdentifier (INTO OUTFILE filename)? (FORMAT identifier)? # ExistsDatabaseStmt + | EXISTS (DICTIONARY | TEMPORARY? TABLE | VIEW)? tableIdentifier (INTO OUTFILE filename)? (FORMAT identifier)? # ExistsTableStmt ; // EXPLAIN statement explainStmt - : EXPLAIN AST query # ExplainASTStmt - | EXPLAIN SYNTAX query # ExplainSyntaxStmt + : EXPLAIN (AST | SYNTAX | QUERY TREE | PLAN | PIPELINE | ESTIMATE | TABLE OVERRIDE)? settingExprList? .+? + | EXPLAIN .+? ; // INSERT statement diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java index da5534341..09329cc16 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java @@ -136,11 +136,8 @@ public void setHasErrors(boolean hasErrors) { @Override public void enterQueryStmt(ClickHouseParser.QueryStmtContext ctx) { - ClickHouseParser.QueryContext qCtx = ctx.query(); - if (qCtx != null) { - if (qCtx.selectStmt() != null || qCtx.selectUnionStmt() != null || qCtx.showStmt() != null || qCtx.describeStmt() != null || qCtx.ctes() != null) { - setHasResultSet(true); - } + if (SqlParser.isStmtWithResultSet(ctx)) { + setHasResultSet(true); } } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java index 425af8e84..63b4230d5 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java @@ -78,12 +78,8 @@ public void visitErrorNode(ErrorNode node) { @Override public void enterQueryStmt(ClickHouseParser.QueryStmtContext ctx) { - ClickHouseParser.QueryContext qCtx = ctx.query(); - if (qCtx != null) { - if (qCtx.selectStmt() != null || qCtx.selectUnionStmt() != null || qCtx.showStmt() != null - || qCtx.describeStmt() != null) { - setHasResultSet(true); - } + if (SqlParser.isStmtWithResultSet(ctx)) { + setHasResultSet(true); } } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java index 6812ffc66..4c13c40e2 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java @@ -48,4 +48,12 @@ public void syntaxError(Recognizer recognizer, Object offendingSymbol, int LOG.warn("SQL syntax error at line: " + line + ", pos: " + charPositionInLine + ", " + msg); } } + + static boolean isStmtWithResultSet(ClickHouseParser.QueryStmtContext stmtContext) { + ClickHouseParser.QueryContext qCtx = stmtContext.query(); + + return qCtx != null && (qCtx.selectStmt() != null || qCtx.selectUnionStmt() != null || + qCtx.showStmt() != null || qCtx.explainStmt() != null || qCtx.describeStmt() != null || + qCtx.existsStmt() != null || qCtx.checkStmt() != null); + } } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 7cc5863ff..274a62213 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -492,12 +492,12 @@ public static Object[][] testStatementWithoutResultSetDP() { {"SHOW FUNCTIONS LIKE '%max%'", 0, true}, {"SHOW MERGES", 0, true}, {"SHOW MERGES LIKE 'your_t%' LIMIT 1", 0, true}, - {"EXPLAIN SELECT sum(number) FROM numbers(10) GROUP BY number % ?;", 1, true}, - {"EXPLAIN SELECT sum(number) FROM numbers(10) GROUP BY number % 4;", 0, true}, + {"EXPLAIN SELECT sum(number) FROM numbers(10) GROUP BY number", 0, true}, + {"EXPLAIN SELECT 1", 0, true}, + {"EXPLAIN SELECT sum(number) FROM numbers(10) UNION ALL SELECT sum(number) FROM numbers(10) ORDER BY sum(number) ASC FORMAT TSV", 0, true}, {"DESCRIBE TABLE table", 0, true}, {"DESC TABLE table1", 0, true}, {"EXISTS TABLE `db`.`table01`", 0, true}, - {"EXISTS TABLE ?", 1, true}, {"CHECK GRANT SELECT(col2) ON table_2", 0, true}, {"CHECK TABLE test_table", 0, true}, {"CHECK TABLE t0 PARTITION ID '201003' FORMAT PrettyCompactMonoBlock SETTINGS check_query_single_value_result = 0", 0, true}, @@ -512,8 +512,9 @@ public static Object[][] testStatementWithoutResultSetDP() { {"CREATE DATABASE IF NOT EXISTS `test_db` ON CLUSTER `cluster` ENGINE = Replicated('clickhouse1:9000', 'test_db')", 0, false}, {"CREATE TABLE `test_table` (id UInt64)", 0, false}, {"CREATE TABLE IF NOT EXISTS `test_table` (id UInt64)", 0, false}, - {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id", 0, false}, - {"CREATE TABLE `test_table` (id UInt64 NOT NULL ) ENGINE = MergeTree() ORDER BY id", 0, false}, + {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree ORDER BY (id)", 0, false}, + {"CREATE TABLE `test_table` (id UInt64) ENGINE = Memory", 0, false}, + {"CREATE TABLE `test_table` (id UInt64 NOT NULL ) ENGINE = MergeTree ORDER BY id", 0, false}, {"CREATE TABLE `test_table` (id UInt64 NULL ) ENGINE = MergeTree() ORDER BY id", 0, false}, {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster`", 0, false}, {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster` ENGINE = Replicated('clickhouse1:9000', 'test_db')", 0, false}, @@ -524,10 +525,9 @@ public static Object[][] testStatementWithoutResultSetDP() { {"CREATE VIEW `test_db`.`source_table` source AS ( SELECT * FROM source_a UNION SELECT * FROM source_b) ENGINE = MaterializedView", 0, false}, {"CREATE VIEW `test_db`.`source_table` source AS ( SELECT * FROM source_a UNION SELECT * FROM source_b) ENGINE = MaterializedView()", 0, false}, {"CREATE VIEW `test_db`.`source_table` source AS ( SELECT * FROM source_a UNION SELECT * FROM source_b) ENGINE = MaterializedView() COMMENT 'for tests'", 0, false}, - {"CREATE DICTIONARY `test_db`.dict1 (k1 UInt64 EXPRESSION(k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000)", 0, false}, - {"CREATE DICTIONARY `test_db`.dict1 (k1 UInt64 (k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000) SETTINGS cache_size = 1000 COMMENT 'for tests'", 0, false}, - {"CREATE OR REPLACE DICTIONARY IF NOT EXISTS `test_db`.dict1 (k1 UInt64 (k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000) SETTINGS cache_size = 1000 COMMENT 'for tests'", 0, false}, - {"CREATE OR REPLACE DICTIONARY IF NOT EXISTS `dict1` (k1 UInt64 (k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000) SETTINGS cache_size = 1000 COMMENT 'for tests'", 0, false}, + { "CREATE DICTIONARY `test_db`.dict1 (k1 UInt64 EXPRESSION(k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000)", 0, false}, + {"CREATE DICTIONARY `test_db`.dict1 (k1 UInt64 (k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000) SETTINGS(cache_size = 1000) COMMENT 'for tests'", 0, false}, + {"CREATE OR REPLACE DICTIONARY IF NOT EXISTS `test_db`.dict1 (k1 UInt64 (k1 + 1), k2 String DEFAULT 'default', a1 Array(UInt64) DEFAULT []) PRIMARY KEY k1 SOURCE(CLICKHOUSE(db='test_db', table='dict1')) LAYOUT(FLAT()) LIFETIME(MIN 1000 MAX 2000) SETTINGS(cache_size = 1000, v='123') COMMENT 'for tests'", 0, false}, {"CREATE FUNCTION test_func AS () -> 10", 0, false}, {"CREATE FUNCTION test_func AS (x) -> 10 * x", 0, false}, {"CREATE FUNCTION test_func AS (x, y) -> y * x", 0, false}, @@ -542,10 +542,9 @@ public static Object[][] testStatementWithoutResultSetDP() { {"CREATE QUOTA qB FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default", 0, false}, {"CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin", 0, false}, {"CREATE NAMED COLLECTION foobar AS a = '1', b = '2' OVERRIDABLE", 0, false}, - {"ALTER TABLE table1 ALTER COLUMN value Int64", 0, false}, - {"alter table t alter column j default 1", 0, false}, - {"alter table t modify comment 'comment'", 0, false}, - +// {"alter table t alter column j default 1", 0, false}, // not supported by CH + {"ALTER TABLE t MODIFY COLUMN j default 1", 0, false}, + {"ALTER TABLE t MODIFY COMMENT 'comment'", 0, false}, {"DELETE FROM db.table1 ON CLUSTER `default` WHERE max(a, 10) > ?", 1, false}, {"DELETE FROM table WHERE a = ?", 1, false}, {"DELETE FROM table WHERE a = ? AND b = ?", 2, false}, From b2d2399f6f9d1d5a83b095c99acd904992abf09f Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 7 Oct 2025 11:10:36 -0700 Subject: [PATCH 07/30] fixed GRANT stmts and added REVOKE statements --- .../jdbc/internal/parser/ClickHouseLexer.g4 | 1 + .../jdbc/internal/parser/ClickHouseParser.g4 | 48 +++++++++++++++++-- .../jdbc/internal/SqlParserTest.java | 7 +-- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 index 444460994..ba4d096b3 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 @@ -237,6 +237,7 @@ PROFILE : P R O F I L E; PROFILES : P R O F I L E S; PROJECTION : P R O J E C T I O N; PULLING : P U L L I N G ; +REVOKE : R E V O K E; QUARTER : Q U A R T E R; QUEUE : Q U E U E ; QUEUES : Q U E U E S ; diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index bfb893a09..7fbf6e390 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -33,10 +33,13 @@ query | showStmt | systemStmt | truncateStmt // DDL + | deleteStmt + | updateStmt | useStmt | watchStmt | ctes? selectStmt | grantStmt + | revokeStmt ; // CTE statement @@ -59,6 +62,17 @@ cteUnboundCol | LPAREN ctes? selectStmt RPAREN AS identifier # CteUnboundNestedSelect ; +// DELETE statement + +deleteStmt + : DELETE FROM tableIdentifier clusterClause? (IN partitionClause)? whereClause? + ; + +// UPDATE statement +updateStmt + : UPDATE tableIdentifier clusterClause? SET assignmentExprList whereClause? + ; + // ALTER statement alterStmt @@ -66,9 +80,9 @@ alterStmt ; alterTableClause - : ADD COLUMN (IF NOT EXISTS)? tableColumnDfnt (AFTER nestedIdentifier)? # AlterTableClauseAddColumn - | ADD INDEX (IF NOT EXISTS)? tableIndexDfnt (AFTER nestedIdentifier)? # AlterTableClauseAddIndex - | ADD PROJECTION (IF NOT EXISTS)? tableProjectionDfnt (AFTER nestedIdentifier)? # AlterTableClauseAddProjection + : ADD COLUMN (IF NOT EXISTS)? tableColumnDfnt (AFTER (nestedIdentifier | FIRST))? # AlterTableClauseAddColumn + | ADD INDEX (IF NOT EXISTS)? tableIndexDfnt (AFTER (nestedIdentifier | FIRST))? # AlterTableClauseAddIndex + | ADD PROJECTION (IF NOT EXISTS)? tableProjectionDfnt (AFTER (nestedIdentifier | FIRST))? # AlterTableClauseAddProjection | ATTACH partitionClause (FROM tableIdentifier)? # AlterTableClauseAttach | CLEAR COLUMN (IF EXISTS)? nestedIdentifier (IN partitionClause)? # AlterTableClauseClearColumn | CLEAR INDEX (IF EXISTS)? nestedIdentifier (IN partitionClause)? # AlterTableClauseClearIndex @@ -87,6 +101,7 @@ alterTableClause | MODIFY COLUMN (IF EXISTS)? nestedIdentifier COMMENT STRING_LITERAL # AlterTableClauseModifyComment | MODIFY COLUMN (IF EXISTS)? nestedIdentifier REMOVE tableColumnPropertyType # AlterTableClauseModifyRemove | MODIFY COLUMN (IF EXISTS)? tableColumnDfnt # AlterTableClauseModify + | ALTER COLUMN (IF EXISTS)? identifier TYPE columnTypeExpr codecExpr? ttlClause? settingExprList? (AFTER (nestedIdentifier | FIRST))? # AlterTableClauseAlterType | MODIFY ORDER BY columnExpr # AlterTableClauseModifyOrderBy | MODIFY ttlClause # AlterTableClauseModifyTTL | MODIFY COMMENT literal # AlterTableClauseModifyComment @@ -107,6 +122,7 @@ assignmentExprList assignmentExpr : nestedIdentifier EQ_SINGLE columnExpr + | nestedIdentifier EQ_SINGLE QUERY ; tableColumnPropertyType @@ -608,10 +624,25 @@ setRolesList : identifier (COMMA identifier)* ; +// GRANT statements + grantStmt - : GRANT clusterClause? ((privilege ON grantTableIdentifier) | (identifier (COMMA identifier)*)) + : GRANT clusterClause? identifier (COMMA identifier)* TO (CURRENT_USER | identifier (COMMA identifier)*) + (WITH ADMIN OPTION)? (WITH REPLACE OPTION)? + | GRANT clusterClause? privelegeList ON grantTableIdentifier TO (CURRENT_USER | identifier) (COMMA identifier)* (WITH GRANT OPTION)? (WITH REPLACE OPTION)? + | GRANT CURRENT GRANTS (LPAREN ((privelegeList ON grantTableIdentifier) | (identifier (COMMA identifier)*)) RPAREN)? + TO (CURRENT_USER | identifier (COMMA identifier)*) + (WITH GRANT OPTION)? (WITH REPLACE OPTION)? + ; + +// REVOKE statements +revokeStmt + : REVOKE clusterClause? privelegeList ON grantTableIdentifier + FROM ((CURRENT_USER | identifier) (COMMA identifier)* | ALL | ALL EXCEPT (CURRENT_USER | identifier) (COMMA identifier)* ) + | REVOKE clusterClause? (ADMIN OPTION FOR)? identifier (COMMA identifier)* + FROM ((CURRENT_USER | identifier) (COMMA identifier)* | ALL | ALL EXCEPT (CURRENT_USER | identifier) (COMMA identifier)* ) ; grantTableIdentifier @@ -621,6 +652,15 @@ grantTableIdentifier | (ASTERISK DOT)? ASTERISK ; +privelegeList + : columnPrivilege (COMMA columnPrivilege)* + ; + + +columnPrivilege + : privilege (LPAREN identifier (COMMA identifier)* RPAREN)? + ; + privilege : | ACCESS MANAGEMENT diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 274a62213..350ed1bcf 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -542,7 +542,7 @@ public static Object[][] testStatementWithoutResultSetDP() { {"CREATE QUOTA qB FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default", 0, false}, {"CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin", 0, false}, {"CREATE NAMED COLLECTION foobar AS a = '1', b = '2' OVERRIDABLE", 0, false}, -// {"alter table t alter column j default 1", 0, false}, // not supported by CH + {"alter table t2 alter column v type Int32", 0, false}, {"ALTER TABLE t MODIFY COLUMN j default 1", 0, false}, {"ALTER TABLE t MODIFY COMMENT 'comment'", 0, false}, {"DELETE FROM db.table1 ON CLUSTER `default` WHERE max(a, 10) > ?", 1, false}, @@ -555,10 +555,10 @@ public static Object[][] testStatementWithoutResultSetDP() { {"SYSTEM RELOAD DICTIONARIES ON CLUSTER `default`", 0, false}, {"GRANT SELECT ON db.* TO john", 0, false}, {"GRANT ON CLUSTER `default` SELECT(a, b) ON db1.tableA TO `user` WITH GRANT OPTION WITH REPLACE OPTION", 0, false}, - {"GRANT SELECT db.* TO user01 WITH REPLACE OPTION", 0, false}, + {"GRANT SELECT ON db.* TO user01 WITH REPLACE OPTION", 0, false}, {"GRANT ON CLUSTER role1, role2 TO `user01` WITH ADMIN OPTION WITH REPLACE OPTION", 0, false}, {"GRANT CURRENT GRANTS TO user01", 0, false}, - {"REVOKE SELECT(a,b) ON db1.tableA FROM `user01", 0, false}, + {"REVOKE SELECT(a,b) ON db1.tableA FROM `user01`", 0, false}, {"REVOKE SELECT ON db1.* FROM ALL", 0, false}, {"REVOKE SELECT ON db1.* FROM ALL EXCEPT `admin01`", 0, false}, {"REVOKE SELECT ON db1.* FROM ALL EXCEPT CURRENT USER", 0, false}, @@ -606,6 +606,7 @@ public static Object[][] testStatementWithoutResultSetDP() { {"EXCHANGE DICTIONARIES dict1 AND dict2", 0, false}, {"EXCHANGE DICTIONARIES dict1 AND dict2 ON CLUSTER `default`", 0, false}, {"SET profile = 'profile-name-from-the-settings-file'", 0, false}, + {"SET use_some_feature_flag", 0, false}, {"SET ROLE role1", 0, false}, {"SET DEFAULT ROLE role1 TO user", 0, false}, {"SET DEFAULT ROLE NONE TO user", 0, false}, From dfc938ca62e0fc6a04892e8f447ef712b3fb1145 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 7 Oct 2025 12:25:31 -0700 Subject: [PATCH 08/30] fixed more statement tests --- .../jdbc/internal/parser/ClickHouseLexer.g4 | 2 ++ .../jdbc/internal/parser/ClickHouseParser.g4 | 28 +++++++++++++------ .../jdbc/internal/SqlParserTest.java | 8 ++++-- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 index ba4d096b3..5d4cf1915 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 @@ -100,6 +100,7 @@ ENGINES : E N G I N E S; ESTIMATE : E S T I M A T E; EVENTS : E V E N T S; EXCEPT : E X C E P T; +EXCHANGE : E X C H A N G E; EXISTS : E X I S T S; EXPLAIN : E X P L A I N; EXPRESSION : E X P R E S S I O N; @@ -241,6 +242,7 @@ REVOKE : R E V O K E; QUARTER : Q U A R T E R; QUEUE : Q U E U E ; QUEUES : Q U E U E S ; +QUERY_SQL : Q U E R Y; // conflicts with '?' QUOTA : Q U O T A; QUOTAS : Q U O T A S ; RABBITMQ : R A B B I T M Q; diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index 7fbf6e390..9bc25b7bb 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -40,6 +40,7 @@ query | ctes? selectStmt | grantStmt | revokeStmt + | exchangeStmt ; // CTE statement @@ -368,10 +369,12 @@ describeStmt // DROP statement dropStmt - : (DETACH | DROP) DATABASE (IF EXISTS)? databaseIdentifier clusterClause? # DropDatabaseStmt - | (DETACH | DROP) (DICTIONARY | TEMPORARY? TABLE | VIEW | ROLE | USER) (IF EXISTS)? tableIdentifier clusterClause? ( - NO DELAY - )? # DropTableStmt + : (DETACH | DROP) DATABASE (IF EXISTS)? databaseIdentifier clusterClause? SYNC? + | (DETACH | DROP) (DICTIONARY | TEMPORARY? TABLE | VIEW) (IF EXISTS)? tableIdentifier clusterClause? + (NO DELAY)? SYNC? + | (DETACH | DROP) (USER | ROLE | QUOTA | SETTINGS? PROFILE) (IF EXISTS)? identifier clusterClause? (FROM identifier)? + | (DETACH | DROP) ROW? POLICY (IF EXISTS)? identifier ON grantTableIdentifier (COMMA grantTableIdentifier)* clusterClause? (FROM identifier)? + | (DETACH | DROP) (FUNCTION | NAMED COLLECTION) (IF EXISTS)? identifier clusterClause? ; // EXISTS statement @@ -419,7 +422,8 @@ assignmentValue // KILL statement killStmt - : KILL MUTATION clusterClause? whereClause (SYNC | ASYNC | TEST)? # KillMutationStmt + : KILL MUTATION clusterClause? whereClause (SYNC | ASYNC | TEST)? (FORMAT identifier)? # KillMutationStmt + | KILL QUERY_SQL clusterClause? whereClause (SYNC | ASYNC | TEST)? (FORMAT identifier)? # KillQueryStmt ; // OPTIMIZE statement @@ -432,6 +436,7 @@ optimizeStmt renameStmt : RENAME TABLE tableIdentifier TO tableIdentifier (COMMA tableIdentifier TO tableIdentifier)* clusterClause? + | RENAME ; // PROJECTION SELECT statement @@ -608,6 +613,12 @@ winFrameBound //rangeClause: RANGE LPAREN (MIN identifier MAX identifier | MAX identifier MIN identifier) RPAREN; +// EXCHANGE statement +exchangeStmt + : EXCHANGE (TABLES|DICTIONARIES) tableIdentifier AND tableIdentifier clusterClause? + ; + + // SET statement setStmt @@ -627,11 +638,9 @@ setRolesList // GRANT statements grantStmt - : GRANT clusterClause? identifier (COMMA identifier)* TO (CURRENT_USER | identifier (COMMA identifier)*) - (WITH ADMIN OPTION)? (WITH REPLACE OPTION)? - | GRANT clusterClause? privelegeList ON grantTableIdentifier + : GRANT clusterClause? ((identifier (COMMA identifier)*) | (privelegeList ON grantTableIdentifier)) TO (CURRENT_USER | identifier) (COMMA identifier)* - (WITH GRANT OPTION)? (WITH REPLACE OPTION)? + (WITH ADMIN OPTION)? (WITH GRANT OPTION)? (WITH REPLACE OPTION)? | GRANT CURRENT GRANTS (LPAREN ((privelegeList ON grantTableIdentifier) | (identifier (COMMA identifier)*)) RPAREN)? TO (CURRENT_USER | identifier (COMMA identifier)*) (WITH GRANT OPTION)? (WITH REPLACE OPTION)? @@ -1250,6 +1259,7 @@ keyword | PRECEDING | PREWHERE | PRIMARY + | PROFILE | RANGE | RELOAD | REMOVE diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 350ed1bcf..c5f18c0f1 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -556,7 +556,8 @@ public static Object[][] testStatementWithoutResultSetDP() { {"GRANT SELECT ON db.* TO john", 0, false}, {"GRANT ON CLUSTER `default` SELECT(a, b) ON db1.tableA TO `user` WITH GRANT OPTION WITH REPLACE OPTION", 0, false}, {"GRANT SELECT ON db.* TO user01 WITH REPLACE OPTION", 0, false}, - {"GRANT ON CLUSTER role1, role2 TO `user01` WITH ADMIN OPTION WITH REPLACE OPTION", 0, false}, + {"GRANT ON CLUSTER `default` role1, role2 TO `user01` WITH ADMIN OPTION WITH REPLACE OPTION", 0, false}, + {"GRANT role1, role2 TO `user01` WITH ADMIN OPTION WITH REPLACE OPTION", 0, false}, {"GRANT CURRENT GRANTS TO user01", 0, false}, {"REVOKE SELECT(a,b) ON db1.tableA FROM `user01`", 0, false}, {"REVOKE SELECT ON db1.* FROM ALL", 0, false}, @@ -577,7 +578,9 @@ public static Object[][] testStatementWithoutResultSetDP() { {"DROP TABLE `db1`.`table01`", 0, false}, {"DROP DICTIONARY `dict1`", 0, false}, {"DROP ROLE IF EXISTS `role01`", 0 , false}, - {"DROP POLICY IF EXISTS `pol1`", 0, false}, + {"DROP POLICY IF EXISTS `pol1` ON db1.table1 ON CLUSTER `default` FROM `test`", 0, false}, + {"DROP POLICY IF EXISTS `pol1` ON db1.table1 ON CLUSTER `default`", 0, false}, + {"DROP POLICY IF EXISTS `pol1` ON table1", 0, false}, {"DROP QUOTA IF EXISTS q1", 0, false}, {"DROP SETTINGS PROFILE IF EXISTS `profile1` ON CLUSTER `default`", 0, false}, {"DROP VIEW view1 ON CLUSTER `default` SYNC", 0, false}, @@ -607,6 +610,7 @@ public static Object[][] testStatementWithoutResultSetDP() { {"EXCHANGE DICTIONARIES dict1 AND dict2 ON CLUSTER `default`", 0, false}, {"SET profile = 'profile-name-from-the-settings-file'", 0, false}, {"SET use_some_feature_flag", 0, false}, + {"SET use_some_feature_flag = 'true'", 0, false}, {"SET ROLE role1", 0, false}, {"SET DEFAULT ROLE role1 TO user", 0, false}, {"SET DEFAULT ROLE NONE TO user", 0, false}, From aefc7c9d23e0f92911a6e77431745116d3758ec7 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 7 Oct 2025 13:11:19 -0700 Subject: [PATCH 09/30] added MOVE and UNDROP statements --- .../jdbc/internal/parser/ClickHouseParser.g4 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index 9bc25b7bb..7345065e8 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -22,6 +22,7 @@ query | createStmt // DDL | describeStmt | dropStmt // DDL + | undropStmt // DDL | existsStmt | explainStmt | killStmt // DDL @@ -41,6 +42,7 @@ query | grantStmt | revokeStmt | exchangeStmt + | moveStmt ; // CTE statement @@ -360,6 +362,11 @@ ttlExpr : columnExpr (DELETE | TO DISK STRING_LITERAL | TO VOLUME STRING_LITERAL)? ; +// MOVE statement +moveStmt + : MOVE (USER | ROLE | QUOTA | SETTINGS PROFILE | ROW POLICY) identifier TO identifier + ; + // DESCRIBE statement describeStmt @@ -377,6 +384,10 @@ dropStmt | (DETACH | DROP) (FUNCTION | NAMED COLLECTION) (IF EXISTS)? identifier clusterClause? ; +undropStmt + : UNDROP TABLE tableIdentifier uuidClause? clusterClause? + ; + // EXISTS statement existsStmt From 4efa2c19e252a6fb18ca72c941634367d902b735 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 7 Oct 2025 14:18:09 -0700 Subject: [PATCH 10/30] Fixed quota statements --- .../jdbc/internal/parser/ClickHouseLexer.g4 | 5 +++++ .../jdbc/internal/parser/ClickHouseParser.g4 | 12 ++++++++++++ .../com/clickhouse/jdbc/internal/SqlParserTest.java | 2 ++ 3 files changed, 19 insertions(+) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 index 5d4cf1915..4ffbe8417 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 @@ -163,6 +163,7 @@ KAFKA : K A F K A; KERBEROS : K E R B E R O S; KEY : K E Y; KEYS : K E Y S; +KEYED : K E Y E D; KILL : K I L L; LAST : L A S T; LAYOUT : L A Y O U T; @@ -172,6 +173,7 @@ LEFT : L E F T; LIFETIME : L I F E T I M E; LIKE : L I K E; LIMIT : L I M I T; +LIMITS : L I M I T S; LISTEN : L I S T E N ; LIVE : L I V E; LOADING : L O A D I N G; @@ -210,6 +212,7 @@ NULL_SQL : N U L L; // conflicts with macro NULL ODBC : O D B C; OFFSET : O F F S E T; ON : O N; +ONLY : O N L Y; OPTIMIZE : O P T I M I Z E; OPTION : O P T I O N; ORDER : O R D E R; @@ -238,6 +241,7 @@ PROFILE : P R O F I L E; PROFILES : P R O F I L E S; PROJECTION : P R O J E C T I O N; PULLING : P U L L I N G ; +RANDOMIZED : R A N D O M I Z E D; REVOKE : R E V O K E; QUARTER : Q U A R T E R; QUEUE : Q U E U E ; @@ -317,6 +321,7 @@ TIMESTAMP : T I M E S T A M P; TOP : T O P; TOTALS : T O T A L S; TO : T O; +TRACKING : T R A C K I N G; TRAILING : T R A I L I N G; TRANSACTION : T R A N S A C T I O N; TREE : T R E E; diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index 7345065e8..7eb024919 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -190,6 +190,18 @@ createStmt | ( INHERIT identifier))? (TO identifier | ALL | ALL EXCEPT identifier)? # createProfileStmt | CREATE FUNCTION identifier clusterClause? AS LPAREN (identifier)? (COMMA identifier)? RPAREN ARROW .+? #createFunctionStmt | CREATE NAMED COLLECTION (IF NOT EXISTS)? identifier clusterClause? AS nameCollectionKey (COMMA nameCollectionKey)* #createNamedCollectionStmt + | CREATE QUOTA (IF NOT EXISTS | OR REPLACE)? identifier clusterClause? (IN identifier)? + (KEYED BY identifier | NOT KEYED)? + quotaForClause (COMMA quotaForClause)* + (TO (identifier (COMMA identifier)* | ALL | CURRENT_USER | ALL EXCEPT identifier (COMMA identifier)* ))? # createQuotaStmt + ; + +quotaMaxExpr + : identifier EQ_SINGLE numberLiteral + ; + +quotaForClause + : FOR RANDOMIZED? INTERVAL numberLiteral interval (MAX quotaMaxExpr (COMMA quotaMaxExpr)*)+? ; nameCollectionKey diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index c5f18c0f1..6b99349cf 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -538,6 +538,8 @@ public static Object[][] testStatementWithoutResultSetDP() { {"CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter", 0, false}, {"CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 TO peter, antonio", 0, false}, {"CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio", 0, false}, + {"CREATE QUOTA qA FOR INTERVAL 15 month MAX queries = 123 TO role1, role2", 0, false}, + {"CREATE QUOTA qA FOR INTERVAL 15 month MAX queries = 123 TO ALL EXCEPT role3", 0, false}, {"CREATE QUOTA qA FOR INTERVAL 15 month MAX queries = 123 TO CURRENT_USER", 0, false}, {"CREATE QUOTA qB FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default", 0, false}, {"CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin", 0, false}, From 25549e7ded0af35935a6bb5ffdf8520d055574ee Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 7 Oct 2025 16:31:38 -0700 Subject: [PATCH 11/30] Fixed insert statements --- .../com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 | 2 +- .../java/com/clickhouse/jdbc/internal/SqlParserTest.java | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index 7eb024919..aa12352b4 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -439,7 +439,7 @@ assignmentValue : literal # InsertRawValue | QUERY # InsertParameter | identifier (LPAREN columnExprList? RPAREN)? # InsertParameterFuncExpr - | LPAREN columnExpr RPAREN # InserParameterExpr + | LPAREN? columnExpr RPAREN? # InserParameterExpr ; // KILL statement diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 6b99349cf..d2faafa29 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -333,8 +333,8 @@ public Object[][] testMiscStmtDp() { {"WITH toDate('2025-08-20') as DATE_END, events AS ( SELECT 1 ) SELECT * FROM events", 0}, {"select 1 table where 1 = ?", 1}, {"insert into t (i, t) values (1, timestamp '2010-01-01 00:00:00')", 0}, - {"insert into t (i, t) values (1, date '2010-01-01')", 0 - } + {"insert into t (i, t) values (1, date '2010-01-01')", 0}, + {"SELECT timestamp '2010-01-01 00:00:00' as ts, date '2010-01-01' as d", 0}, }; } @@ -422,7 +422,6 @@ public Object[][] testMiscStmtDp() { " EventDate = toDate(?) AND\n" + " EventTime <= ts_upper_bound;"; - @Test(dataProvider = "testStatementWithoutResultSetDP") public void testStatementsForResultSet(String sql, int args, boolean hasResultSet) { SqlParser parser = new SqlParser(); From f9dd217116d2c16d225aad4c2adfb6bbf03b9b8c Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 9 Oct 2025 16:33:38 -0700 Subject: [PATCH 12/30] Fixed CTE and some minor bugs --- .../client/datatypes/DataTypeTests.java | 6 ++- .../clickhouse/client/query/QueryTests.java | 2 +- .../jdbc/internal/parser/ClickHouseParser.g4 | 52 ++++++++++--------- .../clickhouse/jdbc/internal/SqlParser.java | 3 +- .../com/clickhouse/jdbc/DataTypeTests.java | 2 +- .../jdbc/PreparedStatementTest.java | 3 ++ .../jdbc/internal/SqlParserTest.java | 32 +++++++++--- 7 files changed, 63 insertions(+), 37 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index 08c45118a..a309de972 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.lang.reflect.Method; import java.math.BigDecimal; +import java.math.RoundingMode; import java.time.Instant; import java.time.LocalDateTime; import java.time.Period; @@ -539,6 +540,9 @@ public void testDynamicWithPrimitives() throws Exception { case Decimal128: case Decimal256: BigDecimal tmpDec = row.getBigDecimal("field").stripTrailingZeros(); + if (tmpDec.divide((BigDecimal)value, RoundingMode.FLOOR).equals(BigDecimal.ONE)) { + continue; + } strValue = tmpDec.toPlainString(); break; case IntervalMicrosecond: @@ -739,7 +743,7 @@ public void testTime64() throws Exception { return; // time64 was introduced in 25.6 } - String table = "test_time64_type"; + String table = "data_type_tests_time64"; client.execute("DROP TABLE IF EXISTS " + table).get(); client.execute(tableDefinition(table, "o_num UInt32", "t_sec Time64(0)", "t_ms Time64(3)", "t_us Time64(6)", "t_ns Time64(9)"), (CommandSettings) new CommandSettings().serverSetting("allow_experimental_time_time64_type", "1")).get(); diff --git a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java index d3dd90eae..3c7491c48 100644 --- a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java @@ -2146,7 +2146,7 @@ public void testGetDynamicValue() throws Exception { } else if (decision == 1) { return rnd.nextInt(); } else { - return rnd.nextDouble(); + return rnd.nextLong(); } }), 1000); diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index aa12352b4..33e4a546b 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -28,7 +28,6 @@ query | killStmt // DDL | optimizeStmt // DDL | renameStmt // DDL - | selectUnionStmt | setStmt | setRoleStmt | showStmt @@ -38,33 +37,14 @@ query | updateStmt | useStmt | watchStmt - | ctes? selectStmt + | selectStmt + | selectUnionStmt | grantStmt | revokeStmt | exchangeStmt | moveStmt ; -// CTE statement -ctes - : LPAREN? WITH cteUnboundCol? (COMMA cteUnboundCol)* COMMA? namedQuery (COMMA namedQuery)* RPAREN? - ; - -namedQuery - : name = identifier (columnAliases)? AS LPAREN query RPAREN - ; - -columnAliases - : LPAREN identifier (',' identifier)* RPAREN - ; - -cteUnboundCol - : (literal AS identifier) # CteUnboundColLiteral - | (QUERY AS identifier) # CteUnboundColParam - | LPAREN? columnExpr RPAREN? AS identifier # CteUnboundColExpr - | LPAREN ctes? selectStmt RPAREN AS identifier # CteUnboundNestedSelect - ; - // DELETE statement deleteStmt @@ -480,7 +460,7 @@ selectStmtWithParens ; selectStmt - : withClause? SELECT DISTINCT? topClause? columnExprList fromClause? arrayJoinClause? windowClause? prewhereClause? whereClause? groupByClause? ( + : cteClause? SELECT DISTINCT? topClause? columnExprList fromClause? arrayJoinClause? windowClause? prewhereClause? whereClause? groupByClause? ( WITH (CUBE | ROLLUP) )? (WITH TOTALS)? havingClause? orderByClause? limitByClause? limitClause? settingsClause? ; @@ -489,6 +469,27 @@ withClause : WITH columnExprList ; +// CTE statement +cteClause + : WITH cteUnboundCol? (COMMA cteUnboundCol)* COMMA? namedQuery? (COMMA namedQuery)* + ; + + +namedQuery + : identifier (columnAliases)? AS LPAREN? ( selectStmt | selectStmtWithParens | selectUnionStmt) RPAREN? + ; + +columnAliases + : LPAREN identifier (',' identifier)* RPAREN + ; + +cteUnboundCol + : literal AS identifier # CteUnboundColLiteral + | QUERY AS identifier # CteUnboundColParam + | LPAREN? columnExpr RPAREN? AS? identifier? # CteUnboundColExpr +// | LPAREN cteStmt? selectStmt RPAREN AS identifier # CteUnboundNestedSelect + ; + topClause : TOP DECIMAL_LITERAL (WITH TIES)? ; @@ -496,7 +497,7 @@ topClause fromClause : FROM joinExpr | FROM identifier LPAREN QUERY RPAREN - | FROM ctes + | FROM selectStmt | FROM identifier LPAREN viewParam (COMMA viewParam)? RPAREN ; @@ -1294,6 +1295,7 @@ keyword | ROLLUP | ROW | ROWS + | REVOKE | SAMPLE | SELECT | SEMI @@ -1320,6 +1322,7 @@ keyword | TRAILING | TRIM | TRUNCATE + | TRACKING | TO | TOP | TTL @@ -1364,6 +1367,7 @@ keywordForAlias | HOUR | MINUTE | SECOND + | REVOKE ; alias diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java index 4c13c40e2..5800199c6 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java @@ -51,9 +51,8 @@ public void syntaxError(Recognizer recognizer, Object offendingSymbol, int static boolean isStmtWithResultSet(ClickHouseParser.QueryStmtContext stmtContext) { ClickHouseParser.QueryContext qCtx = stmtContext.query(); - return qCtx != null && (qCtx.selectStmt() != null || qCtx.selectUnionStmt() != null || qCtx.showStmt() != null || qCtx.explainStmt() != null || qCtx.describeStmt() != null || - qCtx.existsStmt() != null || qCtx.checkStmt() != null); + qCtx.existsStmt() != null || qCtx.checkStmt() != null ); } } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index dc14eebdb..b44672d70 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -617,7 +617,7 @@ public void testTimeTypes() throws SQLException { Properties createProperties = new Properties(); createProperties.put(ClientConfigProperties.serverSetting("allow_experimental_time_time64_type"), "1"); runQuery("CREATE TABLE test_time64 (order Int8, " - + "time Time('UTC'), time64 Time64(9) " + + "time Time, time64 Time64(9) " + ") ENGINE = MergeTree ORDER BY ()", createProperties); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java index 8d8f3e2fe..b1285c32f 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java @@ -44,6 +44,7 @@ import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; +import static org.testng.Assert.fail; @Test(groups = { "integration" }) public class PreparedStatementTest extends JdbcIntegrationTest { @@ -1337,6 +1338,8 @@ public void testSelectWithTableAliasAsKeyword() throws Exception { Assert.assertEquals(rs.getInt(1), 1000); Assert.assertEquals(rs.getString(2), "test"); } + } catch (Exception e) { + fail("failed at keyword " + keyword, e); } } } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index d2faafa29..086f815a1 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -233,26 +233,42 @@ public static Object[][] testCreateStmtDP() { @Test(dataProvider = "testCTEStmtsDP") public void testCTEStatements(String sql, int args) { + System.out.println(sql); SqlParser parser = new SqlParser(); ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); - Assert.assertFalse(stmt.isHasErrors()); - Assert.assertEquals(stmt.getArgCount(), args); - Assert.assertTrue(stmt.isHasResultSet()); + Assert.assertEquals(stmt.getArgCount(), args, "Args mismatch"); + Assert.assertFalse(stmt.isHasErrors(), "Statement has errors"); + Assert.assertTrue(stmt.isHasResultSet(), "show have a result set"); } @DataProvider public static Object[][] testCTEStmtsDP() { return new Object[][] { - {"with ? as a, ? as b select a, b; -- two CTEs of the first form", 2}, + {"with 'a' as a select 1, a union with 'b' as a all select 2, a", 0}, + {"with 'a' as a select 1, a union all with 'b' as a select 2, a", 0}, + {"with ? as a, ? as b select a, b -- two CTEs of the first form", 2}, + {"with 'a' as a, 'b' as b select a, b -- two CTEs of the first form", 0}, {"with a as (select ?), b as (select 2) select * from a, b; -- two CTEs of the second form", 1}, - {"(with a as (select ?) select * from a);", 1}, - {"with a as (select 1) select * from a; ", 0}, - {"(with ? as a select a);", 1}, + {"(with a as (select ?) select * from a)", 1}, + {"with a as (select ?) select * from a", 1}, + {"with a as (select 1) select * from a", 0}, + {"(select 1)", 0}, + {"(with ? as a select a)", 1}, + {"(with 'a' as a select a)", 0}, + {"with ? as a select a", 1}, + {"with 'a' as a select a", 0}, {"select * from ( with x as ( select 9 ) select * from x );", 0}, {"WITH toDateTime(?) AS target_time SELECT * FROM table", 1}, {"WITH toDateTime('2025-08-20 12:34:56') AS target_time SELECT * FROM table", 0}, {"WITH toDate('2025-08-20') as DATE_END, events AS ( SELECT 1 ) SELECT * FROM events", 0}, - {"WITH toDate(?) as DATE_END, events AS ( SELECT 1 ) SELECT * FROM events", 1} + {"WITH toDate(?) as DATE_END, events AS ( SELECT 1 ) SELECT * FROM events", 1}, + {"WITH ? as a, ? as b, body as ( select ? ) select a, b, body.* from body", 3}, + {"WITH 'a_value' as a, 'b_value' as b, body as ( select 'html' ) select a, b, body.* from body", 0}, + {"WITH 'a_value' as a, 'b_value' as b, body as ( with 'data' as d select d, 'html' ) select a, b, body.* from body", 0}, + {"with date select date, 1 from (select now() date)", 0 }, + {COMPLEX_CTE, 4}, + {"WITH 'date' as const1, 'time' as const2, Tmp1 as (SELECT 1), Tmp2 as (SELECT * FROM Tmp1) SELECT * FROM Tmp2 ", 0}, + {"WITH query1 AS ( WITH 'a' as date1 SELECT * FROM tracking.event WHERE project='a' AND time>=starting_time AND time Date: Tue, 14 Oct 2025 09:33:04 -0700 Subject: [PATCH 13/30] fixed CTE arguments positioning and fixed add column expression --- .../jdbc/internal/parser/ClickHouseParser.g4 | 9 +++++---- .../clickhouse/jdbc/internal/SqlParserTest.java | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index 33e4a546b..78a955be0 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -63,8 +63,8 @@ alterStmt ; alterTableClause - : ADD COLUMN (IF NOT EXISTS)? tableColumnDfnt (AFTER (nestedIdentifier | FIRST))? # AlterTableClauseAddColumn - | ADD INDEX (IF NOT EXISTS)? tableIndexDfnt (AFTER (nestedIdentifier | FIRST))? # AlterTableClauseAddIndex + : ADD COLUMN (IF NOT EXISTS)? tableColumnDfnt ((AFTER nestedIdentifier) | FIRST)? # AlterTableClauseAddColumn + | ADD INDEX (IF NOT EXISTS)? tableIndexDfnt ((AFTER nestedIdentifier) | FIRST)? # AlterTableClauseAddIndex | ADD PROJECTION (IF NOT EXISTS)? tableProjectionDfnt (AFTER (nestedIdentifier | FIRST))? # AlterTableClauseAddProjection | ATTACH partitionClause (FROM tableIdentifier)? # AlterTableClauseAttach | CLEAR COLUMN (IF EXISTS)? nestedIdentifier (IN partitionClause)? # AlterTableClauseClearColumn @@ -84,7 +84,7 @@ alterTableClause | MODIFY COLUMN (IF EXISTS)? nestedIdentifier COMMENT STRING_LITERAL # AlterTableClauseModifyComment | MODIFY COLUMN (IF EXISTS)? nestedIdentifier REMOVE tableColumnPropertyType # AlterTableClauseModifyRemove | MODIFY COLUMN (IF EXISTS)? tableColumnDfnt # AlterTableClauseModify - | ALTER COLUMN (IF EXISTS)? identifier TYPE columnTypeExpr codecExpr? ttlClause? settingExprList? (AFTER (nestedIdentifier | FIRST))? # AlterTableClauseAlterType + | ALTER COLUMN (IF EXISTS)? identifier TYPE columnTypeExpr codecExpr? ttlClause? settingExprList? ((AFTER nestedIdentifier) | FIRST)? # AlterTableClauseAlterType | MODIFY ORDER BY columnExpr # AlterTableClauseModifyOrderBy | MODIFY ttlClause # AlterTableClauseModifyTTL | MODIFY COMMENT literal # AlterTableClauseModifyComment @@ -471,7 +471,7 @@ withClause // CTE statement cteClause - : WITH cteUnboundCol? (COMMA cteUnboundCol)* COMMA? namedQuery? (COMMA namedQuery)* + : WITH (cteUnboundCol | namedQuery) (COMMA (cteUnboundCol | namedQuery))* ; @@ -487,6 +487,7 @@ cteUnboundCol : literal AS identifier # CteUnboundColLiteral | QUERY AS identifier # CteUnboundColParam | LPAREN? columnExpr RPAREN? AS? identifier? # CteUnboundColExpr + | LPAREN selectStmt RPAREN AS identifier # CteUnboundSubQuery // | LPAREN cteStmt? selectStmt RPAREN AS identifier # CteUnboundNestedSelect ; diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 086f815a1..29b149ff1 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -233,7 +233,6 @@ public static Object[][] testCreateStmtDP() { @Test(dataProvider = "testCTEStmtsDP") public void testCTEStatements(String sql, int args) { - System.out.println(sql); SqlParser parser = new SqlParser(); ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertEquals(stmt.getArgCount(), args, "Args mismatch"); @@ -269,6 +268,13 @@ public static Object[][] testCTEStmtsDP() { {COMPLEX_CTE, 4}, {"WITH 'date' as const1, 'time' as const2, Tmp1 as (SELECT 1), Tmp2 as (SELECT * FROM Tmp1) SELECT * FROM Tmp2 ", 0}, {"WITH query1 AS ( WITH 'a' as date1 SELECT * FROM tracking.event WHERE project='a' AND time>=starting_time AND time 10", 0, false}, @@ -562,11 +570,12 @@ public static Object[][] testStatementWithoutResultSetDP() { {"alter table t2 alter column v type Int32", 0, false}, {"ALTER TABLE t MODIFY COLUMN j default 1", 0, false}, {"ALTER TABLE t MODIFY COMMENT 'comment'", 0, false}, + {"ALTER TABLE t ADD COLUMN id Int32 AFTER v", 0, false}, + {"ALTER TABLE t ADD COLUMN id Int32 FIRST", 0, false}, {"DELETE FROM db.table1 ON CLUSTER `default` WHERE max(a, 10) > ?", 1, false}, {"DELETE FROM table WHERE a = ?", 1, false}, {"DELETE FROM table WHERE a = ? AND b = ?", 2, false}, {"DELETE FROM hits WHERE Title LIKE '%hello%';", 0, false}, - {"SYSTEM START FETCHES", 0, false}, {"SYSTEM RELOAD DICTIONARIES", 0, false}, {"SYSTEM RELOAD DICTIONARIES ON CLUSTER `default`", 0, false}, From 07eac76c01c6a4af8b7151703bf8a1b7889e8fc1 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 14 Oct 2025 13:08:14 -0700 Subject: [PATCH 14/30] fixed more statements --- .../jdbc/internal/parser/ClickHouseLexer.g4 | 36 ++++++-- .../jdbc/internal/parser/ClickHouseParser.g4 | 85 +++++++++++++++---- .../jdbc/internal/SqlParserTest.java | 3 +- 3 files changed, 98 insertions(+), 26 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 index 4ffbe8417..12b3734ef 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 @@ -41,11 +41,12 @@ BOTH : B O T H; BY : B Y; CACHE : C A C H E ; CACHES : C A C H E S ; +CANCEL : C A N C E L; CASE : C A S E; CAST : C A S T; +CHANGEABLE_IN_READONLY : C H A N G E A B L E UNDERSCORE I N UNDERSCORE R E A D O N L Y; CHANGED : C H A N G E D; CHECK : C H E C K; -CHANGEABLE_IN_READONLY : C H A N G E A B L E UNDERSCORE I N UNDERSCORE R E A D O N L Y; CLEANUP : C L E A N U P; CLEAR : C L E A R; CLIENT : C L I E N T ; @@ -60,15 +61,17 @@ COLUMN : C O L U M N; COLUMNS : C O L U M N S ; COMMENT : C O M M E N T; COMPILED : C O M P I L E D ; +CONDITION : C O N D I T I O N; CONFIG : C O N F I G ; CONNECTIONS : C O N N E C T I O N S ; -CONSTRAINT : C O N S T R A I N T; CONST : C O N S T; +CONSTRAINT : C O N S T R A I N T; CREATE : C R E A T E; CROSS : C R O S S; CUBE : C U B E; CURRENT : C U R R E N T; CURRENT_USER : C U R R E N T '_' U S E R; +CUSTOM : C U S T O M; DATABASE : D A T A B A S E; DATABASES : D A T A B A S E S; DATE : D A T E; @@ -129,6 +132,7 @@ GRANT : G R A N T; GRANTS : G R A N T S; GRANULARITY : G R A N U L A R I T Y; GROUP : G R O U P; +GRPC : G R P C; HAVING : H A V I N G; HDFS : H D F S; HIERARCHICAL : H I E R A R C H I C A L; @@ -136,6 +140,7 @@ HIVE : H I V E; HOST : H O S T; HOUR : H O U R; HTTP : H T T P; +HTTPS : H T T P S; IDENTIFIED : I D E N T I F I E D; ID : I D; IF : I F; @@ -145,8 +150,8 @@ INDEXES : I N D E X E S; INDEX : I N D E X; INDICES : I N D I C E S; INF : I N F | I N F I N I T Y; -IN : I N; INHERIT : I N H E R I T; +IN : I N; INJECTIVE : I N J E C T I V E; INNER : I N N E R; INSERT : I N S E R T; @@ -161,9 +166,9 @@ JEMALLOC : J E M A L L O C ; JOIN : J O I N; KAFKA : K A F K A; KERBEROS : K E R B E R O S; +KEYED : K E Y E D; KEY : K E Y; KEYS : K E Y S; -KEYED : K E Y E D; KILL : K I L L; LAST : L A S T; LAYOUT : L A Y O U T; @@ -171,6 +176,7 @@ LDAP : L D A P; LEADING : L E A D I N G; LEFT : L E F T; LIFETIME : L I F E T I M E; +LIGHTWEIGHT : L I G H T W E I G H T; LIKE : L I K E; LIMIT : L I M I T; LIMITS : L I M I T S; @@ -211,8 +217,8 @@ NULLS : N U L L S; NULL_SQL : N U L L; // conflicts with macro NULL ODBC : O D B C; OFFSET : O F F S E T; -ON : O N; ONLY : O N L Y; +ON : O N; OPTIMIZE : O P T I M I Z E; OPTION : O P T I O N; ORDER : O R D E R; @@ -233,6 +239,7 @@ POLICIES : P O L I C I E S ; POLICY : P O L I C Y; POPULATE : P O P U L A T E; POSTGRES : P O S T G R E S; +POSTGRESQL : P O S T G R E S Q L; PRECEDING : P R E C E D I N G; PREWHERE : P R E W H E R E; PRIMARY : P R I M A R Y; @@ -240,20 +247,23 @@ PROCESSLIST : P R O C E S S L I S T; PROFILE : P R O F I L E; PROFILES : P R O F I L E S; PROJECTION : P R O J E C T I O N; +PROMETHEUS : P R O M E T H E U S; +PROXY : P R O X Y; PULLING : P U L L I N G ; -RANDOMIZED : R A N D O M I Z E D; -REVOKE : R E V O K E; +PULL : P U L L; QUARTER : Q U A R T E R; +QUERIES : Q U E R I E S; +QUERY_SQL : Q U E R Y; // conflicts with '?' QUEUE : Q U E U E ; QUEUES : Q U E U E S ; -QUERY_SQL : Q U E R Y; // conflicts with '?' QUOTA : Q U O T A; QUOTAS : Q U O T A S ; RABBITMQ : R A B B I T M Q; +RANDOMIZED : R A N D O M I Z E D; RANGE : R A N G E; READINESS : R E A D I N E S S; -REALM : R E A L M; READONLY : R E A D O N L Y; +REALM : R E A L M; REDIS : R E D I S; REDUCE : R E D U C E ; REFRESH : R E F R E S H ; @@ -264,12 +274,14 @@ REMOVE : R E M O V E; RENAME : R E N A M E; REPLACE : R E P L A C E; REPLICA : R E P L I C A; +REPLICAS : R E P L I C A S; REPLICATED : R E P L I C A T E D; REPLICATION : R E P L I C A T I O N ; RESOURCE : R E S O U R C E ; RESTART : R E S T A R T; RESTORE : R E S T O R E ; RESTRICTIVE : R E S T R I C T I V E; +REVOKE : R E V O K E; RIGHT : R I G H T; ROLE : R O L E; ROLES : R O L E S ; @@ -283,6 +295,7 @@ SCRAM_SHA256_HASH : S C R A M '_' S H A '2' '5' '6' '_' H A S H; SCRAM_SHA256_PASSWORD : S C R A M '_' S H A '2' '5' '6' '_' P A S S W O R D; SECOND : S E C O N D; SECRETS : S E C R E T S ; +SECURE : S E C U R E; SECURITY : S E C U R I T Y; SELECT : S E L E C T; SEMI : S E M I; @@ -293,6 +306,7 @@ SETTING : S E T T I N G; SETTINGS : S E T T I N G S; SHA256_HASH : S H A '2' '5' '6' '_' H A S H; SHA256_PASSWORD : S H A '2' '5' '6' '_' P A S S W O R D; +SHARD : S H A R D; SHARDS : S H A R D S; SHOW : S H O W; SHUTDOWN : S H U T D O W N ; @@ -305,12 +319,15 @@ SSL_CERTIFICATE : S S L '_' C E R T I F I C A T E; START : S T A R T; STATISTICS : S T A T I S T I C S ; STOP : S T O P; +STRICT : S T R I C T; SUBSTRING : S U B S T R I N G; SYNC : S Y N C; SYNTAX : S Y N T A X; SYSTEM : S Y S T E M; TABLES : T A B L E S; TABLE : T A B L E; +TAG : T A G; +TCP : T C P; TEMPORARY : T E M P O R A R Y; TEST : T E S T; THEN : T H E N; @@ -359,6 +376,7 @@ WITH : W I T H; WORKLOAD : W O R K L O A D; WRITABLE : W R I T A B L E; YEAR : Y E A R | Y Y Y Y; +ZKPATH : Z K P A T H; JSON_FALSE : 'false'; JSON_TRUE : 'true'; diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index 78a955be0..06d83d65c 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -124,7 +124,9 @@ partitionClause // ATTACH statement attachStmt - : ATTACH DICTIONARY tableIdentifier clusterClause? # AttachDictionaryStmt + : ATTACH TABLE (IF NOT EXISTS)? tableIdentifier clusterClause? + | ATTACH DICTIONARY (IF NOT EXISTS)? tableIdentifier clusterClause? + | ATTACH DATABASE (IF NOT EXISTS)? databaseIdentifier engineExpr? clusterClause? ; // CHECK statement @@ -138,8 +140,8 @@ checkStmt // CREATE statement createStmt - : (ATTACH | CREATE) DATABASE (IF NOT EXISTS)? databaseIdentifier clusterClause? engineExpr? # CreateDatabaseStmt - | (ATTACH | CREATE (OR REPLACE)? | REPLACE) DICTIONARY (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? dictionarySchemaClause + : CREATE DATABASE (IF NOT EXISTS)? databaseIdentifier clusterClause? engineExpr? # CreateDatabaseStmt + | (CREATE (OR REPLACE)? | REPLACE) DICTIONARY (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? dictionarySchemaClause dictionaryEngineClause sourceClause layoutClause lifetimeClause dictionarySettingsClause? (COMMENT literal)? # CreateDictionaryStmt | (ATTACH | CREATE) LIVE VIEW (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? ( WITH TIMEOUT DECIMAL_LITERAL? @@ -148,7 +150,7 @@ createStmt destinationClause | engineClause POPULATE? ) subqueryClause # CreateMaterializedViewStmt - | (ATTACH | CREATE (OR REPLACE)? | REPLACE) TEMPORARY? TABLE (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? tableSchemaClause? + | (ATTACH | CREATE (OR REPLACE)? | REPLACE) TEMPORARY? TABLE (IF NOT EXISTS)? tableIdentifier uuidClause? clusterClause? tableSchemaClause engineClause? subqueryClause? # CreateTableStmt | (ATTACH | CREATE) (OR REPLACE)? VIEW (IF NOT EXISTS)? tableIdentifier alias? uuidClause? clusterClause? tableSchemaClause? subqueryClause # CreateViewStmt @@ -432,7 +434,13 @@ killStmt // OPTIMIZE statement optimizeStmt - : OPTIMIZE TABLE tableIdentifier clusterClause? partitionClause? FINAL? DEDUPLICATE? + : OPTIMIZE TABLE tableIdentifier clusterClause? partitionClause? FINAL? DEDUPLICATE? optimizeByExpr? + ; + +optimizeByExpr + : BY ASTERISK (EXCEPT LPAREN? (identifier (COMMA identifier)*) RPAREN? )? + | BY identifier (COMMA identifier)* + | BY COLUMNS LPAREN literal RPAREN (EXCEPT LPAREN? (identifier (COMMA identifier)*) RPAREN? )? ; // RENAME statement @@ -640,7 +648,7 @@ winFrameBound // EXCHANGE statement exchangeStmt - : EXCHANGE (TABLES|DICTIONARIES) tableIdentifier AND tableIdentifier clusterClause? + : EXCHANGE (TABLES | DICTIONARIES) tableIdentifier AND tableIdentifier clusterClause? ; @@ -917,12 +925,12 @@ systemPrivilege // SHOW statements showStmt - : SHOW CREATE? (TEMPORARY? TABLE | DICTIONARY | VIEW | DATABASE) tableIdentifier (INTO OUTFILE literal)? (FORMAT identifier) # showCreateStmt + : SHOW CREATE? (TEMPORARY? TABLE | DICTIONARY | VIEW | DATABASE) tableIdentifier (INTO OUTFILE literal)? (FORMAT identifier)? # showCreateStmt | SHOW DATABASES (NOT? (LIKE | ILIKE) literal) (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showDatabasesStmt - | SHOW FULL? TEMPORARY? TABLES ((FROM | IN) identifier)? (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showTablesStmt - | SHOW EXTENDED? FULL? COLUMNS ((FROM | IN) identifier (FROM | IN) identifier)? (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showColumnsStmt - | SHOW DICTIONARIES ((FROM | IN) identifier)? (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showDictionariesStmt - | SHOW EXTENDED? (INDEX | INDEXES | INDICES | KEYS ) (FROM | IN) identifier ((FROM | IN) identifier)? (WHERE columnExpr) (INTO OUTFILE filename)? (FORMAT identifier)? # showIndexStmt + | SHOW FULL? TEMPORARY? TABLES showFromDbClause? (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showTablesStmt + | SHOW EXTENDED? FULL? COLUMNS showFromTableFromDbClause? (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showColumnsStmt + | SHOW DICTIONARIES showFromDbClause? (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showDictionariesStmt + | SHOW EXTENDED? (INDEX | INDEXES | INDICES | KEYS ) (FROM | IN) identifier showFromTableFromDbClause? (WHERE columnExpr)? (INTO OUTFILE filename)? (FORMAT identifier)? # showIndexStmt | SHOW PROCESSLIST (INTO OUTFILE filename)? (FORMAT identifier)? # showProcessListStmt | SHOW GRANTS (FOR identifier (COMMA identifier)*)? (WITH IMPLICIT)? FINAL? # showGrantsStmt | SHOW CREATE USER ((identifier (COMMA identifier)*) | CURRENT_USER) # showCreateUserStmt @@ -947,16 +955,60 @@ showStmt | SHOW MERGES (NOT? (LIKE | ILIKE) literal)? (LIMIT numberLiteral)? (INTO OUTFILE filename)? (FORMAT identifier)? # showMergesStmt ; +showFromDbClause + : ((FROM | IN) identifier) + ; + +showFromTableFromDbClause + : ((FROM | IN) identifier) showFromDbClause? + ; + // SYSTEM statements systemStmt : SYSTEM FLUSH DISTRIBUTED tableIdentifier - | SYSTEM FLUSH LOGS - | SYSTEM RELOAD DICTIONARIES + | SYSTEM RELOAD DICTIONARIES clusterClause? identifier? | SYSTEM RELOAD DICTIONARY tableIdentifier - | SYSTEM (START | STOP) (DISTRIBUTED SENDS | FETCHES | TTL? MERGES) tableIdentifier - | SYSTEM (START | STOP) REPLICATED SENDS - | SYSTEM SYNC REPLICA tableIdentifier + | SYSTEM RELOAD MODEL clusterClause? identifier? + | SYSTEM RELOAD FUNCTIONS clusterClause? + | SYSTEM RELOAD FUNCTION clusterClause? identifier + | SYSTEM RELOAD ASYNCHRONOUS METRICS clusterClause? + | SYSTEM DROP DNS CACHE + | SYSTEM DROP MARK CACHE + | SYSTEM DROP REPLICA literal (FROM SHARD literal)? (FROM (TABLE tableIdentifier) | (FROM DATABASE identifier) | (ZKPATH literal))? + | SYSTEM DROP UNCOMPRESSED CACHE + | SYSTEM DROP COMPILED EXPRESSION CACHE + | SYSTEM DROP QUERY CONDITION CACHE + | SYSTEM DROP QUERY CACHE (TAG literal)? + | SYSTEM DROP FORMAT SCHEMA CACHE (FOR literal)? + | SYSTEM FLUSH LOGS + | SYSTEM RELOAD CONFIG clusterClause? + | SYSTEM RELOAD USERS clusterClause? + | SYSTEM SHUTDOWN + | SYSTEM KILL + | SYSTEM (START | FLUSH | STOP) (DISTRIBUTED SENDS? | FETCHES | TTL? MERGES) tableIdentifier clusterClause? settingsClause? + | SYSTEM (START | STOP) LISTEN clusterClause? (QUERIES ALL | QUERIES DEFAULT | QUERIES CUSTOM | TCP | TCP WITH PROXY | TCP SECURE | HTTP | HTTPS | MYSQL | GRPC | POSTGRESQL | PROMETHEUS | CUSTOM literal) + | SYSTEM (START | STOP) MERGES clusterClause? ((ON VOLUME identifier) | tableIdentifier)? + | SYSTEM (START | STOP) TTL MERGES clusterClause? tableIdentifier? + | SYSTEM (START | STOP) MOVES clusterClause? tableIdentifier? + | SYSTEM UNFREEZE WITH NAME literal + | SYSTEM WAIT LOADING PARTS clusterClause? tableIdentifier? + | SYSTEM (START | STOP) FETCHES clusterClause? tableIdentifier? + | SYSTEM (START | STOP) REPLICATED SENDS clusterClause? tableIdentifier? + | SYSTEM (START | STOP) REPLICATION QUEUES clusterClause? tableIdentifier? + | SYSTEM (START | STOP) PULLING REPLICATION LOG clusterClause? tableIdentifier? + | SYSTEM SYNC REPLICA clusterClause? tableIdentifier? (IF EXISTS)? (STRICT | LIGHTWEIGHT | FROM literal | PULL)? + | SYSTEM SYNC DATABASE REPLICA identifier + | SYSTEM RESTART REPLICA clusterClause? tableIdentifier? + | SYSTEM RESTORE DATABASE? REPLICA identifier clusterClause? + | SYSTEM RESTART REPLICAS + | SYSTEM DROP FILESYSTEM CACHE clusterClause? + | SYSTEM SYNC FILE CACHE clusterClause? + | SYSTEM (LOAD | UNLOAD) PRIMARY KEY tableIdentifier? + | SYSTEM REFRESH VIEW tableIdentifier + | SYSTEM REPLICATED? (START | STOP) ((VIEW tableIdentifier) | VIEWS) + | SYSTEM CANCEL VIEW tableIdentifier + | SYSTEM WAIT VIEW tableIdentifier ; // TRUNCATE statements @@ -1343,6 +1395,7 @@ keyword | WHERE | WINDOW | WITH + | QUERIES ; keywordForAlias diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 29b149ff1..2df0032cb 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -448,6 +448,7 @@ public Object[][] testMiscStmtDp() { @Test(dataProvider = "testStatementWithoutResultSetDP") public void testStatementsForResultSet(String sql, int args, boolean hasResultSet) { + System.out.println("sql: " + sql); SqlParser parser = new SqlParser(); { ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); @@ -557,7 +558,7 @@ public static Object[][] testStatementWithoutResultSetDP() { {"CREATE FUNCTION test_func ON CLUSTER `cluster` AS (x, y) -> y * x", 0, false}, {"CREATE USER IF NOT EXISTS `user`", 0, false}, {"CREATE USER IF NOT EXISTS `user` ON CLUSTER `cluster`", 0, false}, - {"CREATE ROLE IF NOT EXISTS `role1` ON CLUSTER", 0, false}, + {"CREATE ROLE IF NOT EXISTS `role1` ON CLUSTER 'cluster'", 0, false}, {"CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter", 0, false}, {"CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 TO peter, antonio", 0, false}, {"CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio", 0, false}, From 0f04adfeaf71b19c28051645e17edfd4e54ede55 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 15 Oct 2025 10:09:07 -0700 Subject: [PATCH 15/30] fixed column position in alter statement and renamed QUERY to JDBC_PARAM_PLACEHOLDER to avoid mistakes --- .../jdbc/internal/parser/ClickHouseLexer.g4 | 4 +-- .../jdbc/internal/parser/ClickHouseParser.g4 | 31 +++++++++++-------- .../internal/ParsedPreparedStatement.java | 8 ++--- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 index 12b3734ef..c8ec160a3 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 @@ -253,7 +253,7 @@ PULLING : P U L L I N G ; PULL : P U L L; QUARTER : Q U A R T E R; QUERIES : Q U E R I E S; -QUERY_SQL : Q U E R Y; // conflicts with '?' +QUERY : Q U E R Y; QUEUE : Q U E U E ; QUEUES : Q U E U E S ; QUOTA : Q U O T A; @@ -460,7 +460,7 @@ LT : '<'; NOT_EQ : '!=' | '<>'; PERCENT : '%'; PLUS : '+'; -QUERY : '?'; +JDBC_PARAM_PLACEHOLDER : '?'; QUOTE_DOUBLE : '"'; QUOTE_SINGLE : '\''; RBRACE : '}'; diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 index 06d83d65c..468b280a9 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 @@ -63,9 +63,9 @@ alterStmt ; alterTableClause - : ADD COLUMN (IF NOT EXISTS)? tableColumnDfnt ((AFTER nestedIdentifier) | FIRST)? # AlterTableClauseAddColumn - | ADD INDEX (IF NOT EXISTS)? tableIndexDfnt ((AFTER nestedIdentifier) | FIRST)? # AlterTableClauseAddIndex - | ADD PROJECTION (IF NOT EXISTS)? tableProjectionDfnt (AFTER (nestedIdentifier | FIRST))? # AlterTableClauseAddProjection + : ADD COLUMN (IF NOT EXISTS)? tableColumnDfnt alterTableColumnPosition? # AlterTableClauseAddColumn + | ADD INDEX (IF NOT EXISTS)? tableIndexDfnt alterTableColumnPosition? # AlterTableClauseAddIndex + | ADD PROJECTION (IF NOT EXISTS)? tableProjectionDfnt alterTableColumnPosition? # AlterTableClauseAddProjection | ATTACH partitionClause (FROM tableIdentifier)? # AlterTableClauseAttach | CLEAR COLUMN (IF EXISTS)? nestedIdentifier (IN partitionClause)? # AlterTableClauseClearColumn | CLEAR INDEX (IF EXISTS)? nestedIdentifier (IN partitionClause)? # AlterTableClauseClearIndex @@ -84,7 +84,7 @@ alterTableClause | MODIFY COLUMN (IF EXISTS)? nestedIdentifier COMMENT STRING_LITERAL # AlterTableClauseModifyComment | MODIFY COLUMN (IF EXISTS)? nestedIdentifier REMOVE tableColumnPropertyType # AlterTableClauseModifyRemove | MODIFY COLUMN (IF EXISTS)? tableColumnDfnt # AlterTableClauseModify - | ALTER COLUMN (IF EXISTS)? identifier TYPE columnTypeExpr codecExpr? ttlClause? settingExprList? ((AFTER nestedIdentifier) | FIRST)? # AlterTableClauseAlterType + | ALTER COLUMN (IF EXISTS)? identifier TYPE columnTypeExpr codecExpr? ttlClause? settingExprList? alterTableColumnPosition? # AlterTableClauseAlterType | MODIFY ORDER BY columnExpr # AlterTableClauseModifyOrderBy | MODIFY ttlClause # AlterTableClauseModifyTTL | MODIFY COMMENT literal # AlterTableClauseModifyComment @@ -99,13 +99,18 @@ alterTableClause | UPDATE assignmentExprList whereClause # AlterTableClauseUpdate ; +alterTableColumnPosition + : (AFTER nestedIdentifier) + | FIRST + ; + assignmentExprList : assignmentExpr (COMMA assignmentExpr)* ; assignmentExpr : nestedIdentifier EQ_SINGLE columnExpr - | nestedIdentifier EQ_SINGLE QUERY + | nestedIdentifier EQ_SINGLE JDBC_PARAM_PLACEHOLDER ; tableColumnPropertyType @@ -419,7 +424,7 @@ assignmentValues assignmentValue : literal # InsertRawValue - | QUERY # InsertParameter + | JDBC_PARAM_PLACEHOLDER # InsertParameter | identifier (LPAREN columnExprList? RPAREN)? # InsertParameterFuncExpr | LPAREN? columnExpr RPAREN? # InserParameterExpr ; @@ -428,7 +433,7 @@ assignmentValue killStmt : KILL MUTATION clusterClause? whereClause (SYNC | ASYNC | TEST)? (FORMAT identifier)? # KillMutationStmt - | KILL QUERY_SQL clusterClause? whereClause (SYNC | ASYNC | TEST)? (FORMAT identifier)? # KillQueryStmt + | KILL QUERY clusterClause? whereClause (SYNC | ASYNC | TEST)? (FORMAT identifier)? # KillQueryStmt ; // OPTIMIZE statement @@ -493,7 +498,7 @@ columnAliases cteUnboundCol : literal AS identifier # CteUnboundColLiteral - | QUERY AS identifier # CteUnboundColParam + | JDBC_PARAM_PLACEHOLDER AS identifier # CteUnboundColParam | LPAREN? columnExpr RPAREN? AS? identifier? # CteUnboundColExpr | LPAREN selectStmt RPAREN AS identifier # CteUnboundSubQuery // | LPAREN cteStmt? selectStmt RPAREN AS identifier # CteUnboundNestedSelect @@ -505,13 +510,13 @@ topClause fromClause : FROM joinExpr - | FROM identifier LPAREN QUERY RPAREN + | FROM identifier LPAREN JDBC_PARAM_PLACEHOLDER RPAREN | FROM selectStmt | FROM identifier LPAREN viewParam (COMMA viewParam)? RPAREN ; viewParam - : identifier EQ_SINGLE (literal | QUERY) + : identifier EQ_SINGLE (literal | JDBC_PARAM_PLACEHOLDER) ; arrayJoinClause @@ -1096,7 +1101,7 @@ columnExpr | columnExpr OR columnExpr # ColumnExprOr // TODO(ilezhankin): `BETWEEN a AND b AND c` is parsed in a wrong way: `BETWEEN (a AND b) AND c` | columnExpr NOT? BETWEEN columnExpr AND columnExpr # ColumnExprBetween - | columnExpr QUERY columnExpr COLON columnExpr # ColumnExprTernaryOp + | columnExpr JDBC_PARAM_PLACEHOLDER columnExpr COLON columnExpr # ColumnExprTernaryOp | columnExpr (alias | AS identifier) # ColumnExprAlias | (tableIdentifier DOT)? ASTERISK # ColumnExprAsterisk // single-column only | LPAREN selectUnionStmt RPAREN # ColumnExprSubquery // single-column only @@ -1104,13 +1109,13 @@ columnExpr | LPAREN columnExprList RPAREN # ColumnExprTuple | LBRACKET columnExprList? RBRACKET # ColumnExprArray | columnIdentifier # ColumnExprIdentifier - | QUERY (CAST_OP identifier)? # ColumnExprParam + | JDBC_PARAM_PLACEHOLDER (CAST_OP identifier)? # ColumnExprParam | columnExpr REGEXP literal # ColumnExprRegexp ; columnArgList : columnArgExpr (COMMA columnArgExpr)* - | QUERY (COMMA QUERY)* + | JDBC_PARAM_PLACEHOLDER (COMMA JDBC_PARAM_PLACEHOLDER)* ; columnArgExpr diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java index 09329cc16..a5971876f 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java @@ -199,15 +199,15 @@ public void enterInsertParameter(ClickHouseParser.InsertParameterContext ctx) { @Override public void enterFromClause(ClickHouseParser.FromClauseContext ctx) { - if (ctx.QUERY() != null) { - appendParameter(ctx.QUERY().getSymbol().getStartIndex()); + if (ctx.JDBC_PARAM_PLACEHOLDER() != null) { + appendParameter(ctx.JDBC_PARAM_PLACEHOLDER().getSymbol().getStartIndex()); } } @Override public void enterViewParam(ClickHouseParser.ViewParamContext ctx) { - if (ctx.QUERY() != null) { - appendParameter(ctx.QUERY().getSymbol().getStartIndex()); + if (ctx.JDBC_PARAM_PLACEHOLDER() != null) { + appendParameter(ctx.JDBC_PARAM_PLACEHOLDER().getSymbol().getStartIndex()); } } From 3e185225698f9d7a95d27b6d3ffe22bbfe842a12 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 16 Oct 2025 11:40:59 -0700 Subject: [PATCH 16/30] created a SQL parser facade to implement SQL parser selection --- ...ava => ClickHouseSqlParserFacadeTest.java} | 4 +- .../com/clickhouse/jdbc/ConnectionImpl.java | 8 +- .../internal/ParsedPreparedStatement.java | 153 ++-------- .../jdbc/internal/ParsedStatement.java | 56 +--- .../clickhouse/jdbc/internal/SqlParser.java | 58 ---- .../jdbc/internal/SqlParserFacade.java | 280 ++++++++++++++++++ .../jdbc/internal/Antlr4ParserTest.java | 7 + ...Test.java => BaseSqlParserFacadeTest.java} | 21 +- 8 files changed, 323 insertions(+), 264 deletions(-) rename clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/{ClickHouseSqlParserTest.java => ClickHouseSqlParserFacadeTest.java} (99%) delete mode 100644 jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java create mode 100644 jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java create mode 100644 jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/Antlr4ParserTest.java rename jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/{SqlParserTest.java => BaseSqlParserFacadeTest.java} (98%) diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlParserFacadeTest.java similarity index 99% rename from clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java rename to clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlParserFacadeTest.java index d637c3923..cfcfb211e 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlParserTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/parser/ClickHouseSqlParserFacadeTest.java @@ -20,13 +20,13 @@ import com.clickhouse.client.ClickHouseConfig; -public class ClickHouseSqlParserTest { +public class ClickHouseSqlParserFacadeTest { private ClickHouseSqlStatement[] parse(String sql) { return ClickHouseSqlParser.parse(sql, new ClickHouseConfig()); } private String loadSql(String file) { - InputStream inputStream = ClickHouseSqlParserTest.class.getResourceAsStream("/sqls/" + file); + InputStream inputStream = ClickHouseSqlParserFacadeTest.class.getResourceAsStream("/sqls/" + file); StringBuilder sql = new StringBuilder(); try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java index 40611d191..188a54881 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java @@ -12,7 +12,7 @@ import com.clickhouse.jdbc.internal.FeatureManager; import com.clickhouse.jdbc.internal.JdbcConfiguration; import com.clickhouse.jdbc.internal.ParsedPreparedStatement; -import com.clickhouse.jdbc.internal.SqlParser; +import com.clickhouse.jdbc.internal.SqlParserFacade; import com.clickhouse.jdbc.metadata.DatabaseMetaDataImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +64,7 @@ public class ConnectionImpl implements Connection, JdbcV2Wrapper { private final DatabaseMetaDataImpl metadata; protected final Calendar defaultCalendar; - private final SqlParser sqlParser; + private final SqlParserFacade sqlParser; private Executor networkTimeoutExecutor; @@ -117,7 +117,7 @@ public ConnectionImpl(String url, Properties info) throws SQLException { this.metadata = new DatabaseMetaDataImpl(this, false, url); this.defaultCalendar = Calendar.getInstance(); - this.sqlParser = new SqlParser(); + this.sqlParser = SqlParserFacade.getParser(SqlParserFacade.SQLParser.ANTLR4.name()); // TODO: path config string here this.featureManager = new FeatureManager(this.config); } catch (SQLException e) { throw e; @@ -126,7 +126,7 @@ public ConnectionImpl(String url, Properties info) throws SQLException { } } - public SqlParser getSqlParser() { + public SqlParserFacade getSqlParser() { return sqlParser; } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java index a5971876f..a8599ce51 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java @@ -1,22 +1,12 @@ package com.clickhouse.jdbc.internal; -import com.clickhouse.client.api.sql.SQLUtils; -import com.clickhouse.jdbc.internal.parser.ClickHouseParser; -import com.clickhouse.jdbc.internal.parser.ClickHouseParserBaseListener; -import org.antlr.v4.runtime.tree.ErrorNode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; /** - * Parser listener that collects information for prepared statement. + * Model of parsed statement with parameters */ -public class ParsedPreparedStatement extends ClickHouseParserBaseListener { - private static final Logger LOG = LoggerFactory.getLogger(ParsedPreparedStatement.class); +public final class ParsedPreparedStatement { private String table; @@ -78,10 +68,18 @@ public String[] getInsertColumns() { return insertColumns; } + public void setInsertColumns(String[] insertColumns) { + this.insertColumns = insertColumns; + } + public String getTable() { return table; } + public void setTable(String table) { + this.table = table; + } + public int[] getParamPositions() { return paramPositions; } @@ -98,10 +96,18 @@ public int getAssignValuesListStartPosition() { return assignValuesListStartPosition; } + public void setAssignValuesListStartPosition(int assignValuesListStartPosition) { + this.assignValuesListStartPosition = assignValuesListStartPosition; + } + public int getAssignValuesListStopPosition() { return assignValuesListStopPosition; } + public void setAssignValuesListStopPosition(int assignValuesListStopPosition) { + this.assignValuesListStopPosition = assignValuesListStopPosition; + } + public void setUseDatabase(String useDatabase) { this.useDatabase = useDatabase; } @@ -134,132 +140,11 @@ public void setHasErrors(boolean hasErrors) { this.hasErrors = hasErrors; } - @Override - public void enterQueryStmt(ClickHouseParser.QueryStmtContext ctx) { - if (SqlParser.isStmtWithResultSet(ctx)) { - setHasResultSet(true); - } - } - - @Override - public void enterUseStmt(ClickHouseParser.UseStmtContext ctx) { - if (ctx.databaseIdentifier() != null) { - setUseDatabase(SQLUtils.unquoteIdentifier(ctx.databaseIdentifier().getText())); - } - } - - @Override - public void enterSetRoleStmt(ClickHouseParser.SetRoleStmtContext ctx) { - if (ctx.NONE() != null) { - setRoles(Collections.emptyList()); - } else { - List roles = new ArrayList<>(); - for (ClickHouseParser.IdentifierContext id : ctx.setRolesList().identifier()) { - roles.add(SQLUtils.unquoteIdentifier(id.getText())); - } - setRoles(roles); - } - } - - @Override - public void enterColumnExprParam(ClickHouseParser.ColumnExprParamContext ctx) { - appendParameter(ctx.start.getStartIndex()); - } - - @Override - public void enterColumnExprPrecedence3(ClickHouseParser.ColumnExprPrecedence3Context ctx) { - super.enterColumnExprPrecedence3(ctx); - } - - @Override - public void enterCteUnboundColParam(ClickHouseParser.CteUnboundColParamContext ctx) { - appendParameter(ctx.start.getStartIndex()); - } - - @Override - public void visitErrorNode(ErrorNode node) { - setHasErrors(true); - } - - @Override - public void enterInsertParameterFuncExpr(ClickHouseParser.InsertParameterFuncExprContext ctx) { - setUseFunction(true); - } - - @Override - public void enterAssignmentValuesList(ClickHouseParser.AssignmentValuesListContext ctx) { - assignValuesListStartPosition = ctx.getStart().getStartIndex(); - assignValuesListStopPosition = ctx.getStop().getStopIndex(); - } - - @Override - public void enterInsertParameter(ClickHouseParser.InsertParameterContext ctx) { - appendParameter(ctx.start.getStartIndex()); - } - - @Override - public void enterFromClause(ClickHouseParser.FromClauseContext ctx) { - if (ctx.JDBC_PARAM_PLACEHOLDER() != null) { - appendParameter(ctx.JDBC_PARAM_PLACEHOLDER().getSymbol().getStartIndex()); - } - } - - @Override - public void enterViewParam(ClickHouseParser.ViewParamContext ctx) { - if (ctx.JDBC_PARAM_PLACEHOLDER() != null) { - appendParameter(ctx.JDBC_PARAM_PLACEHOLDER().getSymbol().getStartIndex()); - } - } - - private void appendParameter(int startIndex) { + void appendParameter(int startIndex) { argCount++; if (argCount > paramPositions.length) { paramPositions = Arrays.copyOf(paramPositions, paramPositions.length + 10); } paramPositions[argCount - 1] = startIndex; - if (LOG.isTraceEnabled()) { - LOG.trace("parameter position {}", startIndex); - } - } - - @Override - public void enterTableExprIdentifier(ClickHouseParser.TableExprIdentifierContext ctx) { - if (ctx.tableIdentifier() != null) { - this.table = SQLUtils.unquoteIdentifier(ctx.tableIdentifier().getText()); - } - } - - @Override - public void enterInsertStmt(ClickHouseParser.InsertStmtContext ctx) { - ClickHouseParser.TableIdentifierContext tableId = ctx.tableIdentifier(); - if (tableId != null) { - this.table = SQLUtils.unquoteIdentifier(tableId.getText()); - } - - ClickHouseParser.ColumnsClauseContext columns = ctx.columnsClause(); - if (columns != null) { - List names = columns.nestedIdentifier(); - this.insertColumns = new String[names.size()]; - for (int i = 0; i < names.size(); i++) { - this.insertColumns[i] = names.get(i).getText(); - } - } - - setInsert(true); - } - - @Override - public void enterDataClauseSelect(ClickHouseParser.DataClauseSelectContext ctx) { - setInsertWithSelect(true); - } - - @Override - public void enterDataClauseValues(ClickHouseParser.DataClauseValuesContext ctx) { - setAssignValuesGroups(ctx.assignmentValues().size()); - } - - @Override - public void exitInsertParameterFuncExpr(ClickHouseParser.InsertParameterFuncExprContext ctx) { - setUseFunction(true); } } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java index 63b4230d5..f9eb5cde8 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedStatement.java @@ -1,15 +1,11 @@ package com.clickhouse.jdbc.internal; -import com.clickhouse.client.api.sql.SQLUtils; -import com.clickhouse.jdbc.internal.parser.ClickHouseParser; -import com.clickhouse.jdbc.internal.parser.ClickHouseParserBaseListener; -import org.antlr.v4.runtime.tree.ErrorNode; - -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -public class ParsedStatement extends ClickHouseParserBaseListener { +/** + * Model of parsed statement when no parameters are used. + */ +public final class ParsedStatement { private String useDatabase; @@ -17,8 +13,6 @@ public class ParsedStatement extends ClickHouseParserBaseListener { private boolean insert; - private String insertTableId; - private List roles; private boolean hasErrors; @@ -43,14 +37,6 @@ public boolean isInsert() { return insert; } - public void setInsertTableId(String insertTableId) { - this.insertTableId = insertTableId; - } - - public String getInsertTableId() { - return insertTableId; - } - public String getUseDatabase() { return useDatabase; } @@ -70,38 +56,4 @@ public boolean isHasErrors() { public void setHasErrors(boolean hasErrors) { this.hasErrors = hasErrors; } - - @Override - public void visitErrorNode(ErrorNode node) { - setHasErrors(true); - } - - @Override - public void enterQueryStmt(ClickHouseParser.QueryStmtContext ctx) { - if (SqlParser.isStmtWithResultSet(ctx)) { - setHasResultSet(true); - } - } - - @Override - public void enterUseStmt(ClickHouseParser.UseStmtContext ctx) { - if (ctx.databaseIdentifier() != null) { - setUseDatabase(SQLUtils.unquoteIdentifier(ctx.databaseIdentifier().getText())); - } - } - - @Override - public void enterSetRoleStmt(ClickHouseParser.SetRoleStmtContext ctx) { - if (ctx.NONE() != null) { - setRoles(Collections.emptyList()); - } else { - List roles = new ArrayList<>(); - for (ClickHouseParser.IdentifierContext id : ctx.setRolesList().identifier()) { - roles.add(SQLUtils.unquoteIdentifier(id.getText())); - } - setRoles(roles); - } - } - - } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java deleted file mode 100644 index 5800199c6..000000000 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParser.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.clickhouse.jdbc.internal; - -import com.clickhouse.jdbc.internal.parser.ClickHouseLexer; -import com.clickhouse.jdbc.internal.parser.ClickHouseParser; -import com.clickhouse.jdbc.internal.parser.ClickHouseParserBaseListener; -import org.antlr.v4.runtime.BaseErrorListener; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.tree.IterativeParseTreeWalker; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SqlParser { - - private static final Logger LOG = LoggerFactory.getLogger(SqlParser.class); - - public ParsedStatement parsedStatement(String sql) { - ParsedStatement parserListener = new ParsedStatement(); - walkSql(sql, parserListener); - return parserListener; - } - - public ParsedPreparedStatement parsePreparedStatement(String sql) { - ParsedPreparedStatement parserListener = new ParsedPreparedStatement(); - walkSql(sql, parserListener); - return parserListener; - } - - private ClickHouseParser walkSql(String sql, ClickHouseParserBaseListener listener ) { - CharStream charStream = CharStreams.fromString(sql); - ClickHouseLexer lexer = new ClickHouseLexer(charStream); - ClickHouseParser parser = new ClickHouseParser(new CommonTokenStream(lexer)); - parser.removeErrorListeners(); - parser.addErrorListener(new ParserErrorListener()); - - ClickHouseParser.QueryStmtContext parseTree = parser.queryStmt(); - IterativeParseTreeWalker.DEFAULT.walk(listener, parseTree); - - return parser; - } - - private static class ParserErrorListener extends BaseErrorListener { - @Override - public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { - LOG.warn("SQL syntax error at line: " + line + ", pos: " + charPositionInLine + ", " + msg); - } - } - - static boolean isStmtWithResultSet(ClickHouseParser.QueryStmtContext stmtContext) { - ClickHouseParser.QueryContext qCtx = stmtContext.query(); - return qCtx != null && (qCtx.selectStmt() != null || qCtx.selectUnionStmt() != null || - qCtx.showStmt() != null || qCtx.explainStmt() != null || qCtx.describeStmt() != null || - qCtx.existsStmt() != null || qCtx.checkStmt() != null ); - } -} diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java new file mode 100644 index 000000000..50ea5f18b --- /dev/null +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java @@ -0,0 +1,280 @@ +package com.clickhouse.jdbc.internal; + +import com.clickhouse.client.api.sql.SQLUtils; +import com.clickhouse.jdbc.internal.parser.ClickHouseLexer; +import com.clickhouse.jdbc.internal.parser.ClickHouseParser; +import com.clickhouse.jdbc.internal.parser.ClickHouseParserBaseListener; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.IterativeParseTreeWalker; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public abstract class SqlParserFacade { + + private static final Logger LOG = LoggerFactory.getLogger(SqlParserFacade.class); + + public abstract ParsedStatement parsedStatement(String sql); + + public abstract ParsedPreparedStatement parsePreparedStatement(String sql); + + static final class ANTLR4Parser extends SqlParserFacade { + + @Override + public ParsedStatement parsedStatement(String sql) { + ParsedStatement stmt = new ParsedStatement(); + parseSQL(sql, new ParsedStatementListener(stmt)); + return stmt; + } + + @Override + public ParsedPreparedStatement parsePreparedStatement(String sql) { + ParsedPreparedStatement stmt = new ParsedPreparedStatement(); + parseSQL(sql, new ParsedPreparedStatementListener(stmt)); + return stmt; + } + + private ClickHouseParser parseSQL(String sql, ClickHouseParserBaseListener listener) { + CharStream charStream = CharStreams.fromString(sql); + ClickHouseLexer lexer = new ClickHouseLexer(charStream); + ClickHouseParser parser = new ClickHouseParser(new CommonTokenStream(lexer)); + parser.removeErrorListeners(); + parser.addErrorListener(new ParserErrorListener()); + + ClickHouseParser.QueryStmtContext parseTree = parser.queryStmt(); + IterativeParseTreeWalker.DEFAULT.walk(listener, parseTree); + + return parser; + } + + private static class ParserErrorListener extends BaseErrorListener { + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { + LOG.debug("SQL syntax error at line: " + line + ", pos: " + charPositionInLine + ", " + msg); + } + } + + static boolean isStmtWithResultSet(ClickHouseParser.QueryStmtContext stmtContext) { + ClickHouseParser.QueryContext qCtx = stmtContext.query(); + return qCtx != null && (qCtx.selectStmt() != null || qCtx.selectUnionStmt() != null || + qCtx.showStmt() != null || qCtx.explainStmt() != null || qCtx.describeStmt() != null || + qCtx.existsStmt() != null || qCtx.checkStmt() != null); + } + + private static class ParsedStatementListener extends ClickHouseParserBaseListener { + + private final ParsedStatement parsedStatement; + + public ParsedStatementListener(ParsedStatement parsedStatement) { + this.parsedStatement = parsedStatement; + } + + @Override + public void visitErrorNode(ErrorNode node) { + parsedStatement.setHasErrors(true); + } + + @Override + public void enterQueryStmt(ClickHouseParser.QueryStmtContext ctx) { + if (isStmtWithResultSet(ctx)) { + parsedStatement.setHasResultSet(true); + } + } + + @Override + public void enterUseStmt(ClickHouseParser.UseStmtContext ctx) { + if (ctx.databaseIdentifier() != null) { + parsedStatement.setUseDatabase(SQLUtils.unquoteIdentifier(ctx.databaseIdentifier().getText())); + } + } + + @Override + public void enterSetRoleStmt(ClickHouseParser.SetRoleStmtContext ctx) { + if (ctx.NONE() != null) { + parsedStatement.setRoles(Collections.emptyList()); + } else { + List roles = new ArrayList<>(); + for (ClickHouseParser.IdentifierContext id : ctx.setRolesList().identifier()) { + roles.add(SQLUtils.unquoteIdentifier(id.getText())); + } + parsedStatement.setRoles(roles); + } + } + } + + private static final class ParsedPreparedStatementListener extends ClickHouseParserBaseListener { + + private final ParsedPreparedStatement parsedStatement; + + public ParsedPreparedStatementListener(ParsedPreparedStatement parsedStatement) { + this.parsedStatement = parsedStatement; + } + + @Override + public void enterQueryStmt(ClickHouseParser.QueryStmtContext ctx) { + if (isStmtWithResultSet(ctx)) { + parsedStatement.setHasResultSet(true); + } + } + + @Override + public void enterUseStmt(ClickHouseParser.UseStmtContext ctx) { + if (ctx.databaseIdentifier() != null) { + parsedStatement.setUseDatabase(SQLUtils.unquoteIdentifier(ctx.databaseIdentifier().getText())); + } + } + + @Override + public void enterSetRoleStmt(ClickHouseParser.SetRoleStmtContext ctx) { + if (ctx.NONE() != null) { + parsedStatement.setRoles(Collections.emptyList()); + } else { + List roles = new ArrayList<>(); + for (ClickHouseParser.IdentifierContext id : ctx.setRolesList().identifier()) { + roles.add(SQLUtils.unquoteIdentifier(id.getText())); + } + parsedStatement.setRoles(roles); + } + } + + @Override + public void enterColumnExprParam(ClickHouseParser.ColumnExprParamContext ctx) { + parsedStatement.appendParameter(ctx.start.getStartIndex()); + } + + @Override + public void enterColumnExprPrecedence3(ClickHouseParser.ColumnExprPrecedence3Context ctx) { + super.enterColumnExprPrecedence3(ctx); + } + + @Override + public void enterCteUnboundColParam(ClickHouseParser.CteUnboundColParamContext ctx) { + parsedStatement.appendParameter(ctx.start.getStartIndex()); + } + + @Override + public void visitErrorNode(ErrorNode node) { + parsedStatement.setHasErrors(true); + } + + @Override + public void enterInsertParameterFuncExpr(ClickHouseParser.InsertParameterFuncExprContext ctx) { + parsedStatement.setUseFunction(true); + } + + @Override + public void enterAssignmentValuesList(ClickHouseParser.AssignmentValuesListContext ctx) { + parsedStatement.setAssignValuesListStartPosition(ctx.getStart().getStartIndex()); + parsedStatement.setAssignValuesListStopPosition(ctx.getStop().getStopIndex()); + } + + @Override + public void enterInsertParameter(ClickHouseParser.InsertParameterContext ctx) { + parsedStatement.appendParameter(ctx.start.getStartIndex()); + } + + @Override + public void enterFromClause(ClickHouseParser.FromClauseContext ctx) { + if (ctx.JDBC_PARAM_PLACEHOLDER() != null) { + parsedStatement.appendParameter(ctx.JDBC_PARAM_PLACEHOLDER().getSymbol().getStartIndex()); + } + } + + @Override + public void enterViewParam(ClickHouseParser.ViewParamContext ctx) { + if (ctx.JDBC_PARAM_PLACEHOLDER() != null) { + parsedStatement.appendParameter(ctx.JDBC_PARAM_PLACEHOLDER().getSymbol().getStartIndex()); + } + } + + @Override + public void enterTableExprIdentifier(ClickHouseParser.TableExprIdentifierContext ctx) { + if (ctx.tableIdentifier() != null) { + parsedStatement.setTable(SQLUtils.unquoteIdentifier(ctx.tableIdentifier().getText())); + } + } + + @Override + public void enterInsertStmt(ClickHouseParser.InsertStmtContext ctx) { + ClickHouseParser.TableIdentifierContext tableId = ctx.tableIdentifier(); + if (tableId != null) { + parsedStatement.setTable(SQLUtils.unquoteIdentifier(tableId.getText())); + } + + ClickHouseParser.ColumnsClauseContext columns = ctx.columnsClause(); + if (columns != null) { + List names = columns.nestedIdentifier(); + String[] insertColumns = new String[names.size()]; + for (int i = 0; i < names.size(); i++) { + insertColumns[i] = names.get(i).getText(); + } + parsedStatement.setInsertColumns(insertColumns); + } + + parsedStatement.setInsert(true); + } + + @Override + public void enterDataClauseSelect(ClickHouseParser.DataClauseSelectContext ctx) { + parsedStatement.setInsertWithSelect(true); + } + + @Override + public void enterDataClauseValues(ClickHouseParser.DataClauseValuesContext ctx) { + parsedStatement.setAssignValuesGroups(ctx.assignmentValues().size()); + } + + @Override + public void exitInsertParameterFuncExpr(ClickHouseParser.InsertParameterFuncExprContext ctx) { + parsedStatement.setUseFunction(true); + } + } + } + + public enum SQLParser { + /** + * JavaCC used to determine sql type (SELECT, INSERT, etc.) and extract some information + * Separate procedure parses sql for `?` parameter placeholders. + */ + JAVACC_PARAMS_PARSER, + + /** + * ANTLR4 used to determine sql type (SELECT, INSERT, etc.) and extract some information + * Separate procedure parses sql for `?` parameter placeholders. + */ + ANTLR4_PARAMS_PARSER, + + /** + * ANTLR4 used to determine sql type (SELECT, INSERT, etc.), extract some information + * and determine parameter positions. + */ + ANTLR4 + } + + public static SqlParserFacade getParser(String name) throws SQLException { + try { + SQLParser parserSelection = SQLParser.valueOf(name); + switch (parserSelection) { + case JAVACC_PARAMS_PARSER: + return null; + case ANTLR4_PARAMS_PARSER: + return null; + case ANTLR4: + return new ANTLR4Parser(); + } + throw new SQLException("Unsupported parser: " + parserSelection); + } catch (IllegalArgumentException e) { + throw new SQLException("Unknown parser: " + name); + } + } +} diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/Antlr4ParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/Antlr4ParserTest.java new file mode 100644 index 000000000..605eddd4a --- /dev/null +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/Antlr4ParserTest.java @@ -0,0 +1,7 @@ +package com.clickhouse.jdbc.internal; + +public class Antlr4ParserTest extends BaseSqlParserFacadeTest { + public Antlr4ParserTest() throws Exception { + super(SqlParserFacade.SQLParser.ANTLR4.name()); + } +} diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java similarity index 98% rename from jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java rename to jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index 2df0032cb..5c7662231 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -9,7 +9,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; -public class SqlParserTest { +public abstract class BaseSqlParserFacadeTest { // @Test(groups = {"integration"}) // public void testWithComments() throws Exception { @@ -41,10 +41,14 @@ public class SqlParserTest { // assertEquals(SqlParser.parseStatementType("with data as (SELECT number FROM numbers(100)) select * from data").getType(), SqlParser.StatementType.SELECT); // } + private SqlParserFacade parser; + + public BaseSqlParserFacadeTest(String name) throws Exception { + parser = SqlParserFacade.getParser(name); + } + @Test public void testParseInsertPrepared() throws Exception { - SqlParser parser = new SqlParser(); - String sql = "INSERT INTO \n`table` (id, \nnum1, col3) \nVALUES (?, ?, ?) "; ParsedPreparedStatement parsed = parser.parsePreparedStatement(sql); System.out.println("table: " + parsed.getTable()); @@ -95,7 +99,6 @@ public void testParseInsertPrepared() throws Exception { @Test public void testParseSelectPrepared() throws Exception { // development test - SqlParser parser = new SqlParser(); String sql = "SELECT c1, c2, (true ? 1 : 0 ) as foo FROM tab1 WHERE c3 = ? AND c4 = abs(?)"; ParsedPreparedStatement parsed = parser.parsePreparedStatement(sql); @@ -120,8 +123,6 @@ public void testParseSelectPrepared() throws Exception { @Test public void testPreparedStatementCreateSQL() { - SqlParser parser = new SqlParser(); - String sql = "CREATE TABLE IF NOT EXISTS `with_complex_id` (`v?``1` Int32, " + "\"v?\"\"2\" Int32,`v?\\`3` Int32, \"v?\\\"4\" Int32) ENGINE MergeTree ORDER BY ();"; ParsedPreparedStatement parsed = parser.parsePreparedStatement(sql); @@ -136,7 +137,6 @@ public void testPreparedStatementCreateSQL() { @Test public void testPreparedStatementInsertSQL() { - SqlParser parser = new SqlParser(); String sql = "INSERT INTO `test_stmt_split2` VALUES (1, 'abc'), (2, '?'), (3, '?')"; ParsedPreparedStatement parsed = parser.parsePreparedStatement(sql); @@ -179,7 +179,6 @@ public void testPreparedStatementInsertSQL() { @Test public void testStmtWithCasts() { String sql = "SELECT ?::integer, ?, '?:: integer' FROM table WHERE v = ?::integer"; // CAST(?, INTEGER) - SqlParser parser = new SqlParser(); ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertEquals(stmt.getArgCount(), 3); } @@ -187,7 +186,6 @@ public void testStmtWithCasts() { @Test public void testStmtWithFunction() { String sql = "SELECT `parseDateTimeBestEffort`(?, ?) as dt FROM table WHERE v > `parseDateTimeBestEffort`(?, ?) "; - SqlParser parser = new SqlParser(); ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertEquals(stmt.getArgCount(), 4); } @@ -195,7 +193,6 @@ public void testStmtWithFunction() { @Test public void testStmtWithUUID() { String sql = "select sum(value) from `uuid_filter_db`.`uuid_filter_table` where uuid = ?"; - SqlParser parser = new SqlParser(); ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertEquals(stmt.getArgCount(), 1); Assert.assertFalse(stmt.isHasErrors()); @@ -203,7 +200,6 @@ public void testStmtWithUUID() { @Test(dataProvider = "testCreateStmtDP") public void testCreateStatement(String sql) { - SqlParser parser = new SqlParser(); ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertFalse(stmt.isHasErrors()); } @@ -233,7 +229,6 @@ public static Object[][] testCreateStmtDP() { @Test(dataProvider = "testCTEStmtsDP") public void testCTEStatements(String sql, int args) { - SqlParser parser = new SqlParser(); ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertEquals(stmt.getArgCount(), args, "Args mismatch"); Assert.assertFalse(stmt.isHasErrors(), "Statement has errors"); @@ -280,7 +275,6 @@ public static Object[][] testCTEStmtsDP() { @Test(dataProvider = "testMiscStmtDp") public void testMiscStatements(String sql, int args) { - SqlParser parser = new SqlParser(); ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertEquals(stmt.getArgCount(), args); Assert.assertFalse(stmt.isHasErrors()); @@ -449,7 +443,6 @@ public Object[][] testMiscStmtDp() { @Test(dataProvider = "testStatementWithoutResultSetDP") public void testStatementsForResultSet(String sql, int args, boolean hasResultSet) { System.out.println("sql: " + sql); - SqlParser parser = new SqlParser(); { ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertEquals(stmt.getArgCount(), args); From 9cb77382eb5e7900ceb5ccf45f63257bc3985c50 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 17 Oct 2025 12:19:57 -0700 Subject: [PATCH 17/30] implemented antlr4 two variants --- jdbc-v2/pom.xml | 23 ++ .../parser/{ => antlr4}/ClickHouseLexer.g4 | 0 .../parser/{ => antlr4}/ClickHouseParser.g4 | 0 .../com/clickhouse/jdbc/ConnectionImpl.java | 2 +- .../jdbc/internal/SqlParserFacade.java | 132 +++++-- .../parser/javacc/ClickHouseSqlStatement.java | 364 ++++++++++++++++++ .../parser/javacc/ClickHouseSqlUtils.java | 73 ++++ .../internal/parser/javacc/LanguageType.java | 9 + .../internal/parser/javacc/OperationType.java | 5 + .../internal/parser/javacc/ParseHandler.java | 58 +++ .../internal/parser/javacc/StatementType.java | 54 +++ .../src/main/javacc/ClickHouseSqlParser.jj | 2 +- .../jdbc/internal/Antlr4ParamsParserTest.java | 7 + 13 files changed, 685 insertions(+), 44 deletions(-) rename jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/{ => antlr4}/ClickHouseLexer.g4 (100%) rename jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/{ => antlr4}/ClickHouseParser.g4 (100%) create mode 100644 jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java create mode 100644 jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlUtils.java create mode 100644 jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/LanguageType.java create mode 100644 jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/OperationType.java create mode 100644 jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java create mode 100644 jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/StatementType.java create mode 100644 jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/Antlr4ParamsParserTest.java diff --git a/jdbc-v2/pom.xml b/jdbc-v2/pom.xml index afccb2dd3..585e3cdfa 100644 --- a/jdbc-v2/pom.xml +++ b/jdbc-v2/pom.xml @@ -180,6 +180,29 @@ + + + com.helger.maven + ph-javacc-maven-plugin + ${javacc-plugin.version} + + + jjc + generate-sources + + javacc + + + ${minJdk} + true + com.clickhouse.jdbc.internal.parser.javacc + src/main/javacc + src/main/java + + + + + \ No newline at end of file diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 similarity index 100% rename from jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseLexer.g4 rename to jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 similarity index 100% rename from jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/ClickHouseParser.g4 rename to jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java index 188a54881..ccc3b8d50 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java @@ -117,7 +117,7 @@ public ConnectionImpl(String url, Properties info) throws SQLException { this.metadata = new DatabaseMetaDataImpl(this, false, url); this.defaultCalendar = Calendar.getInstance(); - this.sqlParser = SqlParserFacade.getParser(SqlParserFacade.SQLParser.ANTLR4.name()); // TODO: path config string here + this.sqlParser = SqlParserFacade.getParser(SqlParserFacade.SQLParser.ANTLR4_PARAMS_PARSER.name()); // TODO: path config string here this.featureManager = new FeatureManager(this.config); } catch (SQLException e) { throw e; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java index 50ea5f18b..a207e5abc 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java @@ -1,9 +1,10 @@ package com.clickhouse.jdbc.internal; import com.clickhouse.client.api.sql.SQLUtils; -import com.clickhouse.jdbc.internal.parser.ClickHouseLexer; -import com.clickhouse.jdbc.internal.parser.ClickHouseParser; -import com.clickhouse.jdbc.internal.parser.ClickHouseParserBaseListener; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.jdbc.internal.parser.antlr4.ClickHouseLexer; +import com.clickhouse.jdbc.internal.parser.antlr4.ClickHouseParser; +import com.clickhouse.jdbc.internal.parser.antlr4.ClickHouseParserBaseListener; import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; @@ -28,7 +29,7 @@ public abstract class SqlParserFacade { public abstract ParsedPreparedStatement parsePreparedStatement(String sql); - static final class ANTLR4Parser extends SqlParserFacade { + private static class ANTLR4Parser extends SqlParserFacade { @Override public ParsedStatement parsedStatement(String sql) { @@ -41,10 +42,11 @@ public ParsedStatement parsedStatement(String sql) { public ParsedPreparedStatement parsePreparedStatement(String sql) { ParsedPreparedStatement stmt = new ParsedPreparedStatement(); parseSQL(sql, new ParsedPreparedStatementListener(stmt)); + parseParameters(sql, stmt); return stmt; } - private ClickHouseParser parseSQL(String sql, ClickHouseParserBaseListener listener) { + protected ClickHouseParser parseSQL(String sql, ClickHouseParserBaseListener listener) { CharStream charStream = CharStreams.fromString(sql); ClickHouseLexer lexer = new ClickHouseLexer(charStream); ClickHouseParser parser = new ClickHouseParser(new CommonTokenStream(lexer)); @@ -112,9 +114,9 @@ public void enterSetRoleStmt(ClickHouseParser.SetRoleStmtContext ctx) { } } - private static final class ParsedPreparedStatementListener extends ClickHouseParserBaseListener { + protected static class ParsedPreparedStatementListener extends ClickHouseParserBaseListener { - private final ParsedPreparedStatement parsedStatement; + protected final ParsedPreparedStatement parsedStatement; public ParsedPreparedStatementListener(ParsedPreparedStatement parsedStatement) { this.parsedStatement = parsedStatement; @@ -147,21 +149,11 @@ public void enterSetRoleStmt(ClickHouseParser.SetRoleStmtContext ctx) { } } - @Override - public void enterColumnExprParam(ClickHouseParser.ColumnExprParamContext ctx) { - parsedStatement.appendParameter(ctx.start.getStartIndex()); - } - @Override public void enterColumnExprPrecedence3(ClickHouseParser.ColumnExprPrecedence3Context ctx) { super.enterColumnExprPrecedence3(ctx); } - @Override - public void enterCteUnboundColParam(ClickHouseParser.CteUnboundColParamContext ctx) { - parsedStatement.appendParameter(ctx.start.getStartIndex()); - } - @Override public void visitErrorNode(ErrorNode node) { parsedStatement.setHasErrors(true); @@ -178,24 +170,6 @@ public void enterAssignmentValuesList(ClickHouseParser.AssignmentValuesListConte parsedStatement.setAssignValuesListStopPosition(ctx.getStop().getStopIndex()); } - @Override - public void enterInsertParameter(ClickHouseParser.InsertParameterContext ctx) { - parsedStatement.appendParameter(ctx.start.getStartIndex()); - } - - @Override - public void enterFromClause(ClickHouseParser.FromClauseContext ctx) { - if (ctx.JDBC_PARAM_PLACEHOLDER() != null) { - parsedStatement.appendParameter(ctx.JDBC_PARAM_PLACEHOLDER().getSymbol().getStartIndex()); - } - } - - @Override - public void enterViewParam(ClickHouseParser.ViewParamContext ctx) { - if (ctx.JDBC_PARAM_PLACEHOLDER() != null) { - parsedStatement.appendParameter(ctx.JDBC_PARAM_PLACEHOLDER().getSymbol().getStartIndex()); - } - } @Override public void enterTableExprIdentifier(ClickHouseParser.TableExprIdentifierContext ctx) { @@ -241,22 +215,96 @@ public void exitInsertParameterFuncExpr(ClickHouseParser.InsertParameterFuncExpr } } + private static class ANTLR4AndParamsParser extends ANTLR4Parser { + + @Override + public ParsedPreparedStatement parsePreparedStatement(String sql) { + ParsedPreparedStatement stmt = new ParsedPreparedStatement(); + parseSQL(sql, new ParseStatementAndParamsListener(stmt)); + return stmt; + } + + private static class ParseStatementAndParamsListener extends ParsedPreparedStatementListener { + + public ParseStatementAndParamsListener(ParsedPreparedStatement parsedStatement) { + super(parsedStatement); + } + + @Override + public void enterColumnExprParam(ClickHouseParser.ColumnExprParamContext ctx) { + parsedStatement.appendParameter(ctx.start.getStartIndex()); + } + + + @Override + public void enterCteUnboundColParam(ClickHouseParser.CteUnboundColParamContext ctx) { + parsedStatement.appendParameter(ctx.start.getStartIndex()); + } + + @Override + public void enterInsertParameter(ClickHouseParser.InsertParameterContext ctx) { + parsedStatement.appendParameter(ctx.start.getStartIndex()); + } + + @Override + public void enterFromClause(ClickHouseParser.FromClauseContext ctx) { + if (ctx.JDBC_PARAM_PLACEHOLDER() != null) { + parsedStatement.appendParameter(ctx.JDBC_PARAM_PLACEHOLDER().getSymbol().getStartIndex()); + } + } + + @Override + public void enterViewParam(ClickHouseParser.ViewParamContext ctx) { + if (ctx.JDBC_PARAM_PLACEHOLDER() != null) { + parsedStatement.appendParameter(ctx.JDBC_PARAM_PLACEHOLDER().getSymbol().getStartIndex()); + } + } + } + } + + private static void parseParameters(String originalQuery, ParsedPreparedStatement stmt) { + int len = originalQuery.length(); + for (int i = 0; i < len; i++) { + char ch = originalQuery.charAt(i); + if (ClickHouseUtils.isQuote(ch)) { + i = ClickHouseUtils.skipQuotedString(originalQuery, i, len, ch) - 1; + } else if (ch == '?') { + int idx = ClickHouseUtils.skipContentsUntil(originalQuery, i + 2, len, '?', ':'); + if (idx < len && originalQuery.charAt(idx - 1) == ':' && originalQuery.charAt(idx) != ':' + && originalQuery.charAt(idx - 2) != ':') { + i = idx - 1; + } else { + stmt.appendParameter(i); + } + } else if (ch == ';') { + continue; + } else if (i + 1 < len) { + char nextCh = originalQuery.charAt(i + 1); + if (ch == '-' && nextCh == ch) { + i = ClickHouseUtils.skipSingleLineComment(originalQuery, i + 2, len) - 1; + } else if (ch == '/' && nextCh == '*') { + i = ClickHouseUtils.skipMultiLineComment(originalQuery, i + 2, len) - 1; + } + } + } + } + + public enum SQLParser { /** * JavaCC used to determine sql type (SELECT, INSERT, etc.) and extract some information * Separate procedure parses sql for `?` parameter placeholders. */ - JAVACC_PARAMS_PARSER, + JAVACC, /** - * ANTLR4 used to determine sql type (SELECT, INSERT, etc.) and extract some information - * Separate procedure parses sql for `?` parameter placeholders. + * ANTLR4 used to determine sql type (SELECT, INSERT, etc.) and extract some information and parameters */ ANTLR4_PARAMS_PARSER, /** - * ANTLR4 used to determine sql type (SELECT, INSERT, etc.), extract some information - * and determine parameter positions. + * ANTLR4 used to determine sql type (SELECT, INSERT, etc.), extract some information. + * Separate procedure parses sql for `?` parameter placeholders. */ ANTLR4 } @@ -265,10 +313,10 @@ public static SqlParserFacade getParser(String name) throws SQLException { try { SQLParser parserSelection = SQLParser.valueOf(name); switch (parserSelection) { - case JAVACC_PARAMS_PARSER: + case JAVACC: return null; case ANTLR4_PARAMS_PARSER: - return null; + return new ANTLR4AndParamsParser(); case ANTLR4: return new ANTLR4Parser(); } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java new file mode 100644 index 000000000..50b2045a7 --- /dev/null +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java @@ -0,0 +1,364 @@ +package com.clickhouse.jdbc.internal.parser.javacc; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; + +public class ClickHouseSqlStatement { + public static final String DEFAULT_DATABASE = "system"; + public static final String DEFAULT_TABLE = "unknown"; + + public static final String KEYWORD_DATABASE = "DATABASE"; + public static final String KEYWORD_EXISTS = "EXISTS"; + public static final String KEYWORD_FORMAT = "FORMAT"; + public static final String KEYWORD_REPLACE = "REPLACE"; + public static final String KEYWORD_TOTALS = "TOTALS"; + public static final String KEYWORD_VALUES = "VALUES"; + + public static final String KEYWORD_TABLE_COLUMNS_START = "ColumnsStart"; + public static final String KEYWORD_TABLE_COLUMNS_END = "ColumnsEnd"; + public static final String KEYWORD_VALUES_START = "ValuesStart"; + public static final String KEYWORD_VALUES_END = "ValuesEnd"; + + private final String sql; + private final StatementType stmtType; + private final String cluster; + private final String database; + private final String table; + private final String input; + private final String compressAlgorithm; + private final String compressLevel; + private final String format; + private final String file; + private final List parameters; + private final Map positions; + private final Map settings; + private final Set tempTables; + + public ClickHouseSqlStatement(String sql) { + this(sql, StatementType.UNKNOWN, null, null, null, null, null, null, null, null, null, null, null, null); + } + + public ClickHouseSqlStatement(String sql, StatementType stmtType) { + this(sql, stmtType, null, null, null, null, null, null, null, null, null, null, null, null); + } + + public ClickHouseSqlStatement(String sql, StatementType stmtType, String cluster, String database, String table, + String input, String compressAlgorithm, String compressLevel, String format, String file, + List parameters, Map positions, Map settings, + Set tempTables) { + this.sql = sql; + this.stmtType = stmtType; + + this.cluster = cluster; + this.database = database; + this.table = table == null || table.isEmpty() ? DEFAULT_TABLE : table; + this.input = input; + this.compressAlgorithm = compressAlgorithm; + this.compressLevel = compressLevel; + this.format = format; + this.file = file; + + if (parameters != null && !parameters.isEmpty()) { + this.parameters = Collections.unmodifiableList(parameters); + } else { + this.parameters = Collections.emptyList(); + } + + if (positions != null && !positions.isEmpty()) { + Map p = new HashMap<>(); + for (Entry e : positions.entrySet()) { + String keyword = e.getKey(); + Integer position = e.getValue(); + + if (keyword != null && position != null) { + p.put(keyword, position); + } + } + this.positions = Collections.unmodifiableMap(p); + } else { + this.positions = Collections.emptyMap(); + } + + if (settings != null && !settings.isEmpty()) { + Map s = new LinkedHashMap<>(); + for (Entry e : settings.entrySet()) { + String key = e.getKey(); + String value = e.getValue(); + + if (key != null && value != null) { + s.put(key, String.valueOf(e.getValue())); + } + } + this.settings = Collections.unmodifiableMap(s); + } else { + this.settings = Collections.emptyMap(); + } + + if (tempTables != null && !tempTables.isEmpty()) { + Set s = new LinkedHashSet<>(); + s.addAll(tempTables); + this.tempTables = Collections.unmodifiableSet(s); + } else { + this.tempTables = Collections.emptySet(); + } + } + + public String getSQL() { + return this.sql; + } + + public boolean isRecognized() { + return stmtType != StatementType.UNKNOWN; + } + + public boolean isDDL() { + return this.stmtType.getLanguageType() == LanguageType.DDL; + } + + public boolean isDML() { + return this.stmtType.getLanguageType() == LanguageType.DML; + } + + public boolean isQuery() { + return this.stmtType.getOperationType() == OperationType.READ && !this.hasFile(); + } + + public boolean isMutation() { + return this.stmtType.getOperationType() == OperationType.WRITE || this.hasFile(); + } + + public boolean isTCL() { + return this.stmtType.getLanguageType() == LanguageType.TCL; + } + + public boolean isIdemponent() { + boolean result = this.stmtType.isIdempotent() && !this.hasFile(); + + if (!result) { // try harder + switch (this.stmtType) { + case ATTACH: + case CREATE: + case DETACH: + case DROP: + result = positions.containsKey(KEYWORD_EXISTS) || positions.containsKey(KEYWORD_REPLACE); + break; + + default: + break; + } + } + + return result; + } + + public LanguageType getLanguageType() { + return this.stmtType.getLanguageType(); + } + + public OperationType getOperationType() { + return this.stmtType.getOperationType(); + } + + public StatementType getStatementType() { + return this.stmtType; + } + + public String getCluster() { + return this.cluster; + } + + public String getDatabase() { + return this.database; + } + + public String getDatabaseOrDefault(String database) { + return this.database == null ? (database == null ? DEFAULT_DATABASE : database) : this.database; + } + + public String getTable() { + return this.table; + } + + public String getInput() { + return this.input; + } + + public String getCompressAlgorithm() { + return this.compressAlgorithm; + } + + public String getCompressLevel() { + return this.compressLevel; + } + + public String getFormat() { + return this.format; + } + + public String getFile() { + return this.file; + } + + public String getContentBetweenKeywords(String startKeyword, String endKeyword) { + return getContentBetweenKeywords(startKeyword, endKeyword, 0); + } + + public String getContentBetweenKeywords(String startKeyword, String endKeyword, int startOffset) { + if (startOffset < 0) { + startOffset = 0; + } + Integer startPos = positions.get(startKeyword); + Integer endPos = positions.get(endKeyword); + + String content = ""; + if (startPos != null && endPos != null && startPos + startOffset < endPos) { + content = sql.substring(startPos + startOffset, endPos); + } + + return content; + } + + public boolean containsKeyword(String keyword) { + if (keyword == null || keyword.isEmpty()) { + return false; + } + + return positions.containsKey(keyword.toUpperCase(Locale.ROOT)); + } + + public boolean hasCompressAlgorithm() { + return this.compressAlgorithm != null && !this.compressAlgorithm.isEmpty(); + } + + public boolean hasCompressLevel() { + return this.compressLevel != null && !this.compressLevel.isEmpty(); + } + + public boolean hasFormat() { + return this.format != null && !this.format.isEmpty(); + } + + public boolean hasInput() { + return this.input != null && !this.input.isEmpty(); + } + + public boolean hasFile() { + return this.file != null && !this.file.isEmpty(); + } + + public boolean hasSettings() { + return !this.settings.isEmpty(); + } + + public boolean hasWithTotals() { + return this.positions.containsKey(KEYWORD_TOTALS); + } + + public boolean hasValues() { + return this.positions.containsKey(KEYWORD_VALUES); + } + + public boolean hasTempTable() { + return !this.tempTables.isEmpty(); + } + + public List getParameters() { + return this.parameters; + } + + public int getStartPosition(String keyword) { + int position = -1; + + if (!this.positions.isEmpty() && keyword != null) { + Integer p = this.positions.get(keyword.toUpperCase(Locale.ROOT)); + if (p != null) { + position = p.intValue(); + } + } + + return position; + } + + public int getEndPosition(String keyword) { + int position = getStartPosition(keyword); + + return position != -1 && keyword != null ? position + keyword.length() : position; + } + + public Map getPositions() { + return this.positions; + } + + public Map getSettings() { + return this.settings; + } + + public Set getTempTables() { + return this.tempTables; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append('[').append(stmtType.name()).append(']').append(" cluster=").append(cluster).append(", database=") + .append(database).append(", table=").append(table).append(", input=").append(input) + .append(", compressAlgorithm=").append(compressAlgorithm).append(", compressLevel=") + .append(compressLevel).append(", format=").append(format).append(", outfile=").append(file) + .append(", parameters=").append(parameters).append(", positions=").append(positions) + .append(", settings=").append(settings).append(", tempTables=").append(settings).append("\nSQL:\n") + .append(sql); + + return sb.toString(); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((sql == null) ? 0 : sql.hashCode()); + result = prime * result + ((cluster == null) ? 0 : cluster.hashCode()); + result = prime * result + ((database == null) ? 0 : database.hashCode()); + result = prime * result + table.hashCode(); + result = prime * result + ((input == null) ? 0 : input.hashCode()); + result = prime * result + ((compressAlgorithm == null) ? 0 : compressAlgorithm.hashCode()); + result = prime * result + ((compressLevel == null) ? 0 : compressLevel.hashCode()); + result = prime * result + ((format == null) ? 0 : format.hashCode()); + result = prime * result + ((file == null) ? 0 : file.hashCode()); + result = prime * result + ((stmtType == null) ? 0 : stmtType.hashCode()); + + result = prime * result + parameters.hashCode(); + result = prime * result + positions.hashCode(); + result = prime * result + settings.hashCode(); + result = prime * result + tempTables.hashCode(); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + ClickHouseSqlStatement other = (ClickHouseSqlStatement) obj; + return stmtType == other.stmtType && Objects.equals(sql, other.sql) && Objects.equals(cluster, other.cluster) + && Objects.equals(database, other.database) && Objects.equals(table, other.table) + && Objects.equals(input, other.input) && Objects.equals(compressAlgorithm, other.compressAlgorithm) + && Objects.equals(compressLevel, other.compressLevel) && Objects.equals(format, other.format) + && Objects.equals(file, other.file) && parameters.equals(other.parameters) + && positions.equals(other.positions) && settings.equals(other.settings) + && tempTables.equals(other.tempTables); + } +} diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlUtils.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlUtils.java new file mode 100644 index 000000000..6faf418c4 --- /dev/null +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlUtils.java @@ -0,0 +1,73 @@ +package com.clickhouse.jdbc.internal.parser.javacc; + +public final class ClickHouseSqlUtils { + public static boolean isQuote(char ch) { + return ch == '"' || ch == '\'' || ch == '`'; + } + + /** + * Escape quotes in given string. + * + * @param str string + * @param quote quote to escape + * @return escaped string + */ + public static String escape(String str, char quote) { + if (str == null) { + return str; + } + + int len = str.length(); + StringBuilder sb = new StringBuilder(len + 10).append(quote); + + for (int i = 0; i < len; i++) { + char ch = str.charAt(i); + if (ch == quote || ch == '\\') { + sb.append('\\'); + } + sb.append(ch); + } + + return sb.append(quote).toString(); + } + + /** + * Unescape quoted string. + * + * @param str quoted string + * @return unescaped string + */ + public static String unescape(String str) { + if (str == null || str.isEmpty()) { + return str; + } + + int len = str.length(); + char quote = str.charAt(0); + if (!isQuote(quote) || quote != str.charAt(len - 1)) { // not a quoted string + return str; + } + + StringBuilder sb = new StringBuilder(len = len - 1); + for (int i = 1; i < len; i++) { + char ch = str.charAt(i); + + if (++i >= len) { + sb.append(ch); + } else { + char nextChar = str.charAt(i); + if (ch == '\\' || (ch == quote && nextChar == quote)) { + sb.append(nextChar); + } else { + sb.append(ch); + i--; + } + } + } + + return sb.toString(); + } + + private ClickHouseSqlUtils() { + } +} diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/LanguageType.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/LanguageType.java new file mode 100644 index 000000000..1198930e8 --- /dev/null +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/LanguageType.java @@ -0,0 +1,9 @@ +package com.clickhouse.jdbc.internal.parser.javacc; + +public enum LanguageType { + UNKNOWN, // unknown language + DCL, // data control language + DDL, // data definition language + DML, // data manipulation language + TCL // transaction control language +} diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/OperationType.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/OperationType.java new file mode 100644 index 000000000..06650cade --- /dev/null +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/OperationType.java @@ -0,0 +1,5 @@ +package com.clickhouse.jdbc.internal.parser.javacc; + +public enum OperationType { + UNKNOWN, READ, WRITE +} diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java new file mode 100644 index 000000000..434537ac5 --- /dev/null +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java @@ -0,0 +1,58 @@ +package com.clickhouse.jdbc.internal.parser.javacc; + + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public abstract class ParseHandler { + /** + * Handle macro like "#include('/tmp/template.sql')". + * + * @param name name of the macro + * @param parameters parameters + * @return output of the macro, could be null or empty string + */ + public String handleMacro(String name, List parameters) { + return null; + } + + /** + * Handle parameter. + * + * @param cluster cluster + * @param database database + * @param table table + * @param columnIndex columnIndex(starts from 1 not 0) + * @return parameter value + */ + public String handleParameter(String cluster, String database, String table, int columnIndex) { + return null; + } + + /** + * Hanlde statemenet. + * + * @param sql sql statement + * @param stmtType statement type + * @param cluster cluster + * @param database database + * @param table table + * @param compressAlgorithm compression algorithm + * @param compressLevel compression level + * @param format format + * @param input input + * @param file infile or outfile + * @param parameters positions of parameters + * @param positions keyword positions + * @param settings settings + * @param tempTables temporary tables + * @return sql statement, or null means no change + */ + public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType, String cluster, String database, + String table, String input, String compressAlgorithm, String compressLevel, String format, String file, + List parameters, Map positions, Map settings, + Set tempTables) { + return null; + } +} diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/StatementType.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/StatementType.java new file mode 100644 index 000000000..03c47c884 --- /dev/null +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/StatementType.java @@ -0,0 +1,54 @@ +package com.clickhouse.jdbc.internal.parser.javacc; + +public enum StatementType { + UNKNOWN(LanguageType.UNKNOWN, OperationType.UNKNOWN, false), // unknown statement + ALTER(LanguageType.DDL, OperationType.UNKNOWN, false), // alter statement + ALTER_DELETE(LanguageType.DML, OperationType.WRITE, false), // delete statement + ALTER_UPDATE(LanguageType.DML, OperationType.WRITE, false), // update statement + ATTACH(LanguageType.DDL, OperationType.UNKNOWN, false), // attach statement + CHECK(LanguageType.DDL, OperationType.UNKNOWN, true), // check statement + CREATE(LanguageType.DDL, OperationType.UNKNOWN, false), // create statement + DELETE(LanguageType.DML, OperationType.WRITE, false), // the upcoming light-weight delete statement + DESCRIBE(LanguageType.DDL, OperationType.READ, true), // describe/desc statement + DETACH(LanguageType.DDL, OperationType.UNKNOWN, false), // detach statement + DROP(LanguageType.DDL, OperationType.UNKNOWN, false), // drop statement + EXISTS(LanguageType.DML, OperationType.READ, true), // exists statement + EXPLAIN(LanguageType.DDL, OperationType.READ, true), // explain statement + GRANT(LanguageType.DCL, OperationType.UNKNOWN, true), // grant statement + INSERT(LanguageType.DML, OperationType.WRITE, false), // insert statement + KILL(LanguageType.DCL, OperationType.UNKNOWN, false), // kill statement + OPTIMIZE(LanguageType.DDL, OperationType.UNKNOWN, false), // optimize statement + RENAME(LanguageType.DDL, OperationType.UNKNOWN, false), // rename statement + REVOKE(LanguageType.DCL, OperationType.UNKNOWN, true), // revoke statement + SELECT(LanguageType.DML, OperationType.READ, true), // select statement + SET(LanguageType.DCL, OperationType.UNKNOWN, true), // set statement + SHOW(LanguageType.DDL, OperationType.READ, true), // show statement + SYSTEM(LanguageType.DDL, OperationType.UNKNOWN, false), // system statement + TRUNCATE(LanguageType.DDL, OperationType.UNKNOWN, true), // truncate statement + UPDATE(LanguageType.DML, OperationType.WRITE, false), // the upcoming light-weight update statement + USE(LanguageType.DDL, OperationType.UNKNOWN, true), // use statement + WATCH(LanguageType.DDL, OperationType.UNKNOWN, true), // watch statement + TRANSACTION(LanguageType.TCL, OperationType.WRITE, true); // TCL statement + + private LanguageType langType; + private OperationType opType; + private boolean idempotent; + + StatementType(LanguageType langType, OperationType operationType, boolean idempotent) { + this.langType = langType; + this.opType = operationType; + this.idempotent = idempotent; + } + + LanguageType getLanguageType() { + return this.langType; + } + + OperationType getOperationType() { + return this.opType; + } + + boolean isIdempotent() { + return this.idempotent; + } +} diff --git a/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj b/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj index e3eadced5..adfd49ae8 100644 --- a/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj +++ b/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj @@ -25,7 +25,7 @@ options { PARSER_BEGIN(ClickHouseSqlParser) -package com.clickhouse.jdbc.parser; +package com.clickhouse.jdbc.internal.parser.javacc; import java.io.StringReader; diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/Antlr4ParamsParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/Antlr4ParamsParserTest.java new file mode 100644 index 000000000..69f744783 --- /dev/null +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/Antlr4ParamsParserTest.java @@ -0,0 +1,7 @@ +package com.clickhouse.jdbc.internal; + +public class Antlr4ParamsParserTest extends BaseSqlParserFacadeTest { + public Antlr4ParamsParserTest() throws Exception { + super(SqlParserFacade.SQLParser.ANTLR4_PARAMS_PARSER.name()); + } +} From 0cbf9580bc4cc7b00e5214095a32ad4dec551adc Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 17 Oct 2025 14:16:47 -0700 Subject: [PATCH 18/30] Fixed issue with # comments --- .../com/clickhouse/jdbc/ConnectionImpl.java | 4 +- .../com/clickhouse/jdbc/DriverProperties.java | 10 ++++ .../jdbc/internal/SqlParserFacade.java | 2 +- .../internal/BaseSqlParserFacadeTest.java | 51 ++++++++----------- 4 files changed, 35 insertions(+), 32 deletions(-) diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java index ccc3b8d50..b3238e872 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java @@ -117,7 +117,9 @@ public ConnectionImpl(String url, Properties info) throws SQLException { this.metadata = new DatabaseMetaDataImpl(this, false, url); this.defaultCalendar = Calendar.getInstance(); - this.sqlParser = SqlParserFacade.getParser(SqlParserFacade.SQLParser.ANTLR4_PARAMS_PARSER.name()); // TODO: path config string here + + this.sqlParser = SqlParserFacade.getParser(config.getDriverProperty(DriverProperties.SQL_PARSER.getKey(), + DriverProperties.SQL_PARSER.getDefaultValue())); this.featureManager = new FeatureManager(this.config); } catch (SQLException e) { throw e; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java index 6bae0071b..65b737d43 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java @@ -57,6 +57,16 @@ public enum DriverProperties { * */ USE_MAX_RESULT_ROWS("jdbc_use_max_result_rows", String.valueOf(Boolean.FALSE)), + + /** + * Configures what SQL parser will be used. Choices: + *
    + *
  • ANTLR4 - parser extracts required information but PreparedStatement parameters parsed separately.
    • + *
  • ANTLR4_PARAMS_PARSER - parser extracts required information AND parameter positions.
    • + *
  • JAVACC - parser extracts required information but PreparedStatement parameters parsed separately.
    • + *
+ */ + SQL_PARSER("jdbc_sql_parser", "ANTLR4", List.of("ANTLR4", "ANTLR4_PARAMS_PARSER", "JAVACC")), ; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java index a207e5abc..63dd5ab18 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java @@ -280,7 +280,7 @@ private static void parseParameters(String originalQuery, ParsedPreparedStatemen continue; } else if (i + 1 < len) { char nextCh = originalQuery.charAt(i + 1); - if (ch == '-' && nextCh == ch) { + if ((ch == '-' && nextCh == ch) || (ch == '#')) { i = ClickHouseUtils.skipSingleLineComment(originalQuery, i + 2, len) - 1; } else if (ch == '/' && nextCh == '*') { i = ClickHouseUtils.skipMultiLineComment(originalQuery, i + 2, len) - 1; diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index 5c7662231..42bb3d974 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -11,36 +11,6 @@ public abstract class BaseSqlParserFacadeTest { -// @Test(groups = {"integration"}) -// public void testWithComments() throws Exception { -// assertEquals(SqlParser.parseStatementType(" /* INSERT TESTING */\n SELECT 1 AS num").getType(), SqlParser.StatementType.SELECT); -// assertEquals(SqlParser.parseStatementType("/* SELECT TESTING */\n INSERT INTO test_table VALUES (1)").getType(), SqlParser.StatementType.INSERT); -// assertEquals(SqlParser.parseStatementType("/* INSERT TESTING */\n\n\n UPDATE test_table SET num = 2").getType(), SqlParser.StatementType.UPDATE); -// assertEquals(SqlParser.parseStatementType("-- INSERT TESTING */\n SELECT 1 AS num").getType(), SqlParser.StatementType.SELECT); -// assertEquals(SqlParser.parseStatementType(" -- SELECT TESTING \n -- SELECT AGAIN \n INSERT INTO test_table VALUES (1)").getType(), SqlParser.StatementType.INSERT); -// assertEquals(SqlParser.parseStatementType(" SELECT 42 -- INSERT TESTING").getType(), SqlParser.StatementType.SELECT); -// assertEquals(SqlParser.parseStatementType("#! INSERT TESTING \n SELECT 1 AS num").getType(), SqlParser.StatementType.SELECT); -// assertEquals(SqlParser.parseStatementType("#!INSERT TESTING \n SELECT 1 AS num").getType(), SqlParser.StatementType.SELECT); -// assertEquals(SqlParser.parseStatementType("# INSERT TESTING \n SELECT 1 AS num").getType(), SqlParser.StatementType.SELECT); -// assertEquals(SqlParser.parseStatementType("#INSERT TESTING \n SELECT 1 AS num").getType(), SqlParser.StatementType.SELECT); -// assertEquals(SqlParser.parseStatementType("\nINSERT TESTING \n SELECT 1 AS num").getType(), SqlParser.StatementType.INSERT_INTO_SELECT); -// assertEquals(SqlParser.parseStatementType(" \n INSERT TESTING \n SELECT 1 AS num").getType(), SqlParser.StatementType.INSERT_INTO_SELECT); -// assertEquals(SqlParser.parseStatementType("INSERT INTO t SELECT 1 AS num").getType(), SqlParser.StatementType.INSERT_INTO_SELECT); -// assertEquals(SqlParser.parseStatementType("select 1 AS num").getType(), SqlParser.StatementType.SELECT); -// assertEquals(SqlParser.parseStatementType("insert into test_table values (1)").getType(), SqlParser.StatementType.INSERT); -// assertEquals(SqlParser.parseStatementType("update test_table set num = 2").getType(), SqlParser.StatementType.UPDATE); -// assertEquals(SqlParser.parseStatementType("delete from test_table where num = 2").getType(), SqlParser.StatementType.DELETE); -// assertEquals(SqlParser.parseStatementType("sElEcT 1 AS num").getType(), SqlParser.StatementType.SELECT); -// assertEquals(SqlParser.parseStatementType(null).getType(), SqlParser.StatementType.OTHER); -// assertEquals(SqlParser.parseStatementType("").getType(), SqlParser.StatementType.OTHER); -// assertEquals(SqlParser.parseStatementType(" ").getType(), SqlParser.StatementType.OTHER); -// } -// -// @Test(groups = {"integration"}) -// public void testParseStatementWithClause() throws Exception { -// assertEquals(SqlParser.parseStatementType("with data as (SELECT number FROM numbers(100)) select * from data").getType(), SqlParser.StatementType.SELECT); -// } - private SqlParserFacade parser; public BaseSqlParserFacadeTest(String name) throws Exception { @@ -353,9 +323,30 @@ public Object[][] testMiscStmtDp() { {"insert into t (i, t) values (1, timestamp '2010-01-01 00:00:00')", 0}, {"insert into t (i, t) values (1, date '2010-01-01')", 0}, {"SELECT timestamp '2010-01-01 00:00:00' as ts, date '2010-01-01' as d", 0}, + {INSERT_WITH_COMMENTS, 4}, + {" /* INSERT TESTING ?? */\n SELECT ? AS num", 1}, + {"/* SELECT ? TESTING */\n INSERT INTO test_table VALUES (?)", 1}, + {"/* INSERT ? T??ESTING */\n\n\n UPDATE test_table SET num = ?", 1}, + {"-- INSERT ? TESTING */\n SELECT ? AS num", 1}, + {" -- SELECT ? TESTING \n -- SELECT AGAIN ?\n INSERT INTO test_table VALUES (?)", 1}, + {" SELECT ? -- INSERT ? TESTING", 1}, + {"#! INSERT ? TESTING \n SELECT ? AS num", 1}, + {"#!INSERT ? TESTING \n SELECT ? AS num", 1}, + {"# INSERT ? TESTING \n SELECT ? AS num", 1}, + {"#INSERT ? TESTING \n SELECT ? AS num", 1}, + {"\nINSERT INTO TESTING \n SELECT ? AS num", 1}, + {" \n INSERT INTO TESTING \n SELECT ? AS num", 1}, + {" SELECT '##?0.1' as f, ? as a\n #this is debug \n FROM table", 1}, + {"WITH '#!?0.1' as f, ? as a\n #this is debug \n SELECT * FROM a", 1}, }; } + private static final String INSERT_WITH_COMMENTS = "-- line comment1 ?\n" + + "# line comment2 ?\n" + + "#! line comment3 ?\n" + + "/* block comment ? \n */" + + "INSERT INTO `with_complex_id`(`v?``1`, \"v?\"\"2\",`v?\\`3`, \"v?\\\"4\") VALUES (?, ?, ?, ?);"; + private static final String INSERT_INLINE_DATA = "INSERT INTO `interval_15_XUTLZWBLKMNZZPRZSKRF`.`checkins` (`timestamp`, `id`) " + "VALUES ((`now64`(9) + INTERVAL -225 second), 1)"; From 505272daa3fef931e24628411eb4dd5d246af124 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 20 Oct 2025 10:42:11 -0700 Subject: [PATCH 19/30] Added javaCC parser implementation. not a default. some tests failing --- .../jdbc/internal/SqlParserFacade.java | 74 +++++++- .../parser/javacc/JdbcParseHandler.java | 165 ++++++++++++++++++ .../src/main/javacc/ClickHouseSqlParser.jj | 15 +- .../internal/BaseSqlParserFacadeTest.java | 57 +++--- .../jdbc/internal/JavaCCParserTest.java | 11 ++ 5 files changed, 274 insertions(+), 48 deletions(-) create mode 100644 jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java create mode 100644 jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JavaCCParserTest.java diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java index 63dd5ab18..f109f5586 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java @@ -5,6 +5,10 @@ import com.clickhouse.jdbc.internal.parser.antlr4.ClickHouseLexer; import com.clickhouse.jdbc.internal.parser.antlr4.ClickHouseParser; import com.clickhouse.jdbc.internal.parser.antlr4.ClickHouseParserBaseListener; +import com.clickhouse.jdbc.internal.parser.javacc.ClickHouseSqlParser; +import com.clickhouse.jdbc.internal.parser.javacc.ClickHouseSqlStatement; +import com.clickhouse.jdbc.internal.parser.javacc.JdbcParseHandler; +import com.clickhouse.jdbc.internal.parser.javacc.StatementType; import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; @@ -29,6 +33,74 @@ public abstract class SqlParserFacade { public abstract ParsedPreparedStatement parsePreparedStatement(String sql); + private static class JavaCCParser extends SqlParserFacade { + + @Override + public ParsedStatement parsedStatement(String sql) { + ParsedStatement stmt = new ParsedStatement(); + ClickHouseSqlStatement parsedStmt = parse(sql); + if (parsedStmt.getStatementType() == StatementType.USE) { + stmt.setUseDatabase(parsedStmt.getDatabase()); + } + // TODO: set roles + stmt.setInsert(parsedStmt.getStatementType() == StatementType.INSERT); + stmt.setHasErrors(false); + stmt.setHasResultSet(isStmtWithResultSet(parsedStmt)); + return stmt; + } + + private boolean isStmtWithResultSet(ClickHouseSqlStatement parsedStmt) { + return parsedStmt.getStatementType() == StatementType.SELECT || parsedStmt.getStatementType() == StatementType.SHOW + || parsedStmt.getStatementType() == StatementType.EXPLAIN || parsedStmt.getStatementType() == StatementType.DESCRIBE + || parsedStmt.getStatementType() == StatementType.EXISTS || parsedStmt.getStatementType() == StatementType.CHECK; + + } + + @Override + public ParsedPreparedStatement parsePreparedStatement(String sql) { + ParsedPreparedStatement stmt = new ParsedPreparedStatement(); + ClickHouseSqlStatement parsedStmt = parse(sql); + if (parsedStmt.getStatementType() == StatementType.USE) { + stmt.setUseDatabase(parsedStmt.getDatabase()); + } + stmt.setInsert(parsedStmt.getStatementType() == StatementType.INSERT); + stmt.setHasErrors(false); + stmt.setHasResultSet(isStmtWithResultSet(parsedStmt)); + stmt.setTable(parsedStmt.getTable()); + stmt.setInsertWithSelect(parsedStmt.containsKeyword("SELECT") && (parsedStmt.getStatementType() == StatementType.INSERT)); + + Integer startIndex = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_START); + if (startIndex != null) { + stmt.setAssignValuesGroups(1); + int endIndex = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_END); + stmt.setAssignValuesListStartPosition(startIndex); + stmt.setAssignValuesListStopPosition(endIndex); + String query = parsedStmt.getSQL(); + for (int i = startIndex + 1; i < endIndex; i++) { + char ch = query.charAt(i); + if (ch != '?' && ch != ',' && !Character.isWhitespace(ch)) { + stmt.setUseFunction(true); + break; + } + } + } + + stmt.setUseFunction(false); + parseParameters(sql, stmt); + return stmt; + } + + + public ClickHouseSqlStatement parse(String sql) { + JdbcParseHandler handler = JdbcParseHandler.getInstance(); + ClickHouseSqlStatement[] stmts = ClickHouseSqlParser.parse(sql, handler); + if (stmts.length > 1) { + throw new RuntimeException("More than one SQL statement found: " + sql); + } + return stmts[0]; + } + } + private static class ANTLR4Parser extends SqlParserFacade { @Override @@ -314,7 +386,7 @@ public static SqlParserFacade getParser(String name) throws SQLException { SQLParser parserSelection = SQLParser.valueOf(name); switch (parserSelection) { case JAVACC: - return null; + return new JavaCCParser(); case ANTLR4_PARAMS_PARSER: return new ANTLR4AndParamsParser(); case ANTLR4: diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java new file mode 100644 index 000000000..f6faaa05f --- /dev/null +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java @@ -0,0 +1,165 @@ +package com.clickhouse.jdbc.internal.parser.javacc; + +import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseUtils; + + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class JdbcParseHandler extends ParseHandler { + private static final String SETTING_MUTATIONS_SYNC = "mutations_sync"; + + private static final JdbcParseHandler INSTANCE; + + static { + INSTANCE = new JdbcParseHandler(true, true, true); + }; + + public static JdbcParseHandler getInstance() { + return INSTANCE; + } + + private final boolean allowLocalFile; + private final boolean allowLightWeightDelete; + private final boolean allowLightWeightUpdate; + + private void addMutationSetting(String sql, StringBuilder builder, Map positions, + Map settings, int index) { + boolean hasSetting = settings != null && !settings.isEmpty(); + String setting = hasSetting ? settings.get(SETTING_MUTATIONS_SYNC) : null; + if (setting == null) { + String keyword = "SETTINGS"; + Integer settingsIndex = positions.get(keyword); + + if (settingsIndex == null) { + builder.append(sql.substring(index)).append(" SETTINGS mutations_sync=1"); + if (hasSetting) { + builder.append(','); + } + } else { + builder.append(sql.substring(index, settingsIndex)).append("SETTINGS mutations_sync=1,") + .append(sql.substring(settingsIndex + keyword.length())); + } + } else { + builder.append(sql.substring(index)); + } + } + + private ClickHouseSqlStatement handleDelete(String sql, StatementType stmtType, String cluster, String database, + String table, String input, String compressAlgorithm, String compressLevel, String format, String file, + List parameters, Map positions, Map settings, + Set tempTables) { + StringBuilder builder = new StringBuilder(); + int index = positions.get("DELETE"); + if (index > 0) { + builder.append(sql.substring(0, index)); + } + index = positions.get("FROM"); + Integer whereIdx = positions.get("WHERE"); + if (whereIdx != null) { + builder.append("ALTER TABLE "); + if (!ClickHouseChecker.isNullOrEmpty(database)) { + builder.append('`').append(database).append('`').append('.'); + } + builder.append('`').append(table).append('`').append(" DELETE "); + addMutationSetting(sql, builder, positions, settings, whereIdx); + } else { + builder.append("TRUNCATE TABLE").append(sql.substring(index + 4)); + } + return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null); + } + + private ClickHouseSqlStatement handleUpdate(String sql, StatementType stmtType, String cluster, String database, + String table, String input, String compressAlgorithm, String compressLevel, String format, String file, + List parameters, Map positions, Map settings, + Set tempTables) { + StringBuilder builder = new StringBuilder(); + int index = positions.get("UPDATE"); + if (index > 0) { + builder.append(sql.substring(0, index)); + } + builder.append("ALTER TABLE "); + index = positions.get("SET"); + if (!ClickHouseChecker.isNullOrEmpty(database)) { + builder.append('`').append(database).append('`').append('.'); + } + builder.append('`').append(table).append('`').append(" UPDATE"); // .append(sql.substring(index + 3)); + addMutationSetting(sql, builder, positions, settings, index + 3); + return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null); + } + + private ClickHouseSqlStatement handleInFileForInsertQuery(String sql, StatementType stmtType, String cluster, + String database, String table, String input, String compressAlgorithm, String compressLevel, String format, + String file, List parameters, Map positions, Map settings, + Set tempTables) { + StringBuilder builder = new StringBuilder(sql.length()); + builder.append(sql.substring(0, positions.get("FROM"))); + Integer index = positions.get("SETTINGS"); + if (index == null || index < 0) { + index = positions.get("FORMAT"); + } + if (index != null && index > 0) { + builder.append(sql.substring(index)); + } else { + ClickHouseFormat f = ClickHouseFormat.fromFileName(ClickHouseUtils.unescape(file)); + if (f == null) { + f = (ClickHouseFormat) ClickHouseDefaults.FORMAT.getDefaultValue(); + } + format = f.name(); + builder.append("FORMAT ").append(format); + } + return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null); + } + + private ClickHouseSqlStatement handleOutFileForSelectQuery(String sql, StatementType stmtType, String cluster, + String database, String table, String input, String compressAlgorithm, String compressLevel, String format, + String file, List parameters, Map positions, Map settings, + Set tempTables) { + StringBuilder builder = new StringBuilder(sql.length()); + builder.append(sql.substring(0, positions.get("INTO"))); + Integer index = positions.get("FORMAT"); + if (index != null && index > 0) { + builder.append(sql.substring(index)); + } + return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null); + } + + @Override + public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType, String cluster, String database, + String table, String input, String compressAlgorithm, String compressLevel, String format, String file, + List parameters, Map positions, Map settings, + Set tempTables) { + boolean hasFile = allowLocalFile && !ClickHouseChecker.isNullOrEmpty(file) && file.charAt(0) == '\''; + ClickHouseSqlStatement s = null; + if (stmtType == StatementType.DELETE) { + s = allowLightWeightDelete ? s + : handleDelete(sql, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, + format, file, parameters, positions, settings, tempTables); + } else if (stmtType == StatementType.UPDATE) { + s = allowLightWeightUpdate ? s + : handleUpdate(sql, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, + format, file, parameters, positions, settings, tempTables); + } else if (stmtType == StatementType.INSERT && hasFile) { + s = handleInFileForInsertQuery(sql, stmtType, cluster, database, table, input, compressAlgorithm, + compressLevel, format, file, parameters, positions, settings, tempTables); + } else if (stmtType == StatementType.SELECT && hasFile) { + s = handleOutFileForSelectQuery(sql, stmtType, cluster, database, table, input, compressAlgorithm, + compressLevel, format, file, parameters, positions, settings, tempTables); + } + return s; + } + + private JdbcParseHandler(boolean allowLightWeightDelete, boolean allowLightWeightUpdate, boolean allowLocalFile) { + this.allowLightWeightDelete = allowLightWeightDelete; + this.allowLightWeightUpdate = allowLightWeightUpdate; + this.allowLocalFile = allowLocalFile; + } +} diff --git a/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj b/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj index adfd49ae8..87448c13d 100644 --- a/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj +++ b/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj @@ -51,7 +51,6 @@ public class ClickHouseSqlParser { private final List statements = new ArrayList<>(); - private ClickHouseConfig config; private ParseHandler handler; private int anyArgsListStart = -1; @@ -76,14 +75,7 @@ public class ClickHouseSqlParser { return !(getToken(1).kind == AND && token_source.parentToken == BETWEEN); } - public static ClickHouseSqlStatement[] parse(String sql, ClickHouseConfig config) { - return parse(sql, config, null); - } - - public static ClickHouseSqlStatement[] parse(String sql, ClickHouseConfig config, ParseHandler handler) { - if (config == null) { - config = new ClickHouseConfig(); - } + public static ClickHouseSqlStatement[] parse(String sql, ParseHandler handler) { ClickHouseSqlStatement[] stmts = new ClickHouseSqlStatement[] { new ClickHouseSqlStatement(sql, StatementType.UNKNOWN) }; @@ -92,7 +84,7 @@ public class ClickHouseSqlParser { return stmts; } - ClickHouseSqlParser p = new ClickHouseSqlParser(sql, config, handler); + ClickHouseSqlParser p = new ClickHouseSqlParser(sql, handler); try { stmts = p.sql(); } catch (Exception e) { @@ -106,10 +98,9 @@ public class ClickHouseSqlParser { return stmts; } - public ClickHouseSqlParser(String sql, ClickHouseConfig config, ParseHandler handler) { + public ClickHouseSqlParser(String sql, ParseHandler handler) { this(new StringReader(sql)); - this.config = config; this.handler = handler; } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index 42bb3d974..e3f65d6be 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -105,45 +105,32 @@ public void testPreparedStatementCreateSQL() { } - @Test - public void testPreparedStatementInsertSQL() { + @Test(dataProvider = "testPreparedStatementInsertSQLDP") + public void testPreparedStatementInsertSQL(String sql, int assignGroups, boolean insertWithSelect, int args) { - String sql = "INSERT INTO `test_stmt_split2` VALUES (1, 'abc'), (2, '?'), (3, '?')"; ParsedPreparedStatement parsed = parser.parsePreparedStatement(sql); // TODO: extend test expecting no errors - assertTrue(parsed.isInsert()); - assertFalse(parsed.isHasResultSet()); - assertFalse(parsed.isInsertWithSelect()); - assertEquals(parsed.getAssignValuesGroups(), 3); - - sql = "-- line comment1 ?\n" - + "# line comment2 ?\n" - + "#! line comment3 ?\n" - + "/* block comment ? \n */" - + "INSERT INTO `with_complex_id`(`v?``1`, \"v?\"\"2\",`v?\\`3`, \"v?\\\"4\") VALUES (?, ?, ?, ?);"; - parsed = parser.parsePreparedStatement(sql); - // TODO: extend test expecting no errors - assertTrue(parsed.isInsert()); - assertFalse(parsed.isHasResultSet()); - assertFalse(parsed.isInsertWithSelect()); - assertEquals(parsed.getAssignValuesGroups(), 1); - - sql = "INSERT INTO tt SELECT now(), 10, 20.0, 30"; - parsed = parser.parsePreparedStatement(sql); - // TODO: extend test expecting no errors - assertTrue(parsed.isInsert()); - assertFalse(parsed.isHasResultSet()); - assertTrue(parsed.isInsertWithSelect()); - + assertTrue(parsed.isInsert(), "Should be of insert type"); + assertFalse(parsed.isHasResultSet(), "Should not have result set"); + assertEquals(parsed.isInsertWithSelect(), insertWithSelect, "Insert with select attribute does not match"); + assertEquals(parsed.getAssignValuesGroups(), assignGroups, "Assign values groups do not match"); + assertEquals(parsed.getArgCount(), args, "Args do not match"); + } - sql = "INSERT INTO `users` (`name`, `last_login`, `password`, `id`) VALUES\n" + - " (?, `parseDateTimeBestEffort`(?, ?), ?, 1)\n"; - parsed = parser.parsePreparedStatement(sql); - // TODO: extend test expecting no errors - assertTrue(parsed.isInsert()); - assertFalse(parsed.isHasResultSet()); - assertFalse(parsed.isInsertWithSelect()); - assertEquals(parsed.getAssignValuesGroups(), 1); + @DataProvider + public static Object[][] testPreparedStatementInsertSQLDP() { + return new Object[][] { + {"-- line comment1 ?\n" + + "# line comment2 ?\n" + + "#! line comment3 ?\n" + + "/* block comment ? \n */" + + "INSERT INTO `with_complex_id`(`v?``1`, \"v?\"\"2\",`v?\\`3`, \"v?\\\"4\") VALUES (?, ?, ?, ?);", 1, false, 4}, + { "INSERT INTO `test_stmt_split2` VALUES (1, 'abc'), (2, '?'), (3, '?')", 3, false, 0 }, + { "INSERT INTO `with_complex_id`(`v?``1`, \"v?\"\"2\",`v?\\`3`, \"v?\\\"4\") VALUES (?, ?, ?, ?);", 1, false, 4}, + { "INSERT INTO tt SELECT now(), 10, 20.0, 30", -1, true, 0 }, + { "INSERT INTO `users` (`name`, `last_login`, `password`, `id`) VALUES\n" + + " (?, `parseDateTimeBestEffort`(?, ?), ?, 1)\n", 1, false, 4 }, + }; } @Test diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JavaCCParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JavaCCParserTest.java new file mode 100644 index 000000000..86fcd334a --- /dev/null +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JavaCCParserTest.java @@ -0,0 +1,11 @@ +package com.clickhouse.jdbc.internal; + + +import org.testng.annotations.Ignore; + +@Ignore +public class JavaCCParserTest extends BaseSqlParserFacadeTest { + public JavaCCParserTest() throws Exception { + super(SqlParserFacade.SQLParser.JAVACC.name()); + } +} From fdfc35b02b8ce4a19eae8c1262531037860fd3ff Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 22 Oct 2025 15:55:36 -0700 Subject: [PATCH 20/30] Fixed javacc to support single line comments and some new statements --- jdbc-v2/pom.xml | 3 +-- .../jdbc/internal/ParsedPreparedStatement.java | 2 +- .../jdbc/internal/SqlParserFacade.java | 2 +- .../parser/javacc/ClickHouseSqlStatement.java | 12 +++++++++--- .../parser/javacc/JdbcParseHandler.java | 18 +++++++++--------- .../internal/parser/javacc/ParseHandler.java | 2 +- jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj | 16 +++++++++++----- .../jdbc/internal/BaseSqlParserFacadeTest.java | 7 ++++++- .../jdbc/internal/JavaCCParserTest.java | 2 +- 9 files changed, 40 insertions(+), 24 deletions(-) diff --git a/jdbc-v2/pom.xml b/jdbc-v2/pom.xml index 585e3cdfa..a64c240d5 100644 --- a/jdbc-v2/pom.xml +++ b/jdbc-v2/pom.xml @@ -17,7 +17,7 @@ https://github.com/ClickHouse/clickhouse-java/tree/main/jdbc-v2 - 4.1.4 + 5.0.0 JDBC 4.2 2.17.2 @@ -197,7 +197,6 @@ true com.clickhouse.jdbc.internal.parser.javacc src/main/javacc - src/main/java diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java index a8599ce51..b17db6703 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java @@ -34,7 +34,7 @@ public final class ParsedPreparedStatement { private int assignValuesListStopPosition = -1; - private int assignValuesGroups = -1; + private int assignValuesGroups = 0; public void setHasResultSet(boolean hasResultSet) { this.hasResultSet = hasResultSet; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java index f109f5586..a3a49ecb0 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java @@ -68,10 +68,10 @@ public ParsedPreparedStatement parsePreparedStatement(String sql) { stmt.setHasResultSet(isStmtWithResultSet(parsedStmt)); stmt.setTable(parsedStmt.getTable()); stmt.setInsertWithSelect(parsedStmt.containsKeyword("SELECT") && (parsedStmt.getStatementType() == StatementType.INSERT)); + stmt.setAssignValuesGroups(parsedStmt.getValueGroups()); Integer startIndex = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_START); if (startIndex != null) { - stmt.setAssignValuesGroups(1); int endIndex = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_END); stmt.setAssignValuesListStartPosition(startIndex); stmt.setAssignValuesListStopPosition(endIndex); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java index 50b2045a7..5afad67e7 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java @@ -41,19 +41,20 @@ public class ClickHouseSqlStatement { private final Map positions; private final Map settings; private final Set tempTables; + private final int valueGroups; public ClickHouseSqlStatement(String sql) { - this(sql, StatementType.UNKNOWN, null, null, null, null, null, null, null, null, null, null, null, null); + this(sql, StatementType.UNKNOWN, null, null, null, null, null, null, null, null, null, null, null, null, 0); } public ClickHouseSqlStatement(String sql, StatementType stmtType) { - this(sql, stmtType, null, null, null, null, null, null, null, null, null, null, null, null); + this(sql, stmtType, null, null, null, null, null, null, null, null, null, null, null, null, 0); } public ClickHouseSqlStatement(String sql, StatementType stmtType, String cluster, String database, String table, String input, String compressAlgorithm, String compressLevel, String format, String file, List parameters, Map positions, Map settings, - Set tempTables) { + Set tempTables, int valueGroups) { this.sql = sql; this.stmtType = stmtType; @@ -65,6 +66,7 @@ public ClickHouseSqlStatement(String sql, StatementType stmtType, String cluster this.compressLevel = compressLevel; this.format = format; this.file = file; + this.valueGroups = valueGroups; if (parameters != null && !parameters.isEmpty()) { this.parameters = Collections.unmodifiableList(parameters); @@ -297,6 +299,10 @@ public Map getPositions() { return this.positions; } + public int getValueGroups() { + return valueGroups; + } + public Map getSettings() { return this.settings; } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java index f6faaa05f..4f36c5729 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java @@ -71,7 +71,7 @@ private ClickHouseSqlStatement handleDelete(String sql, StatementType stmtType, builder.append("TRUNCATE TABLE").append(sql.substring(index + 4)); } return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, - compressAlgorithm, compressLevel, format, file, parameters, null, settings, null); + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0); } private ClickHouseSqlStatement handleUpdate(String sql, StatementType stmtType, String cluster, String database, @@ -91,13 +91,13 @@ private ClickHouseSqlStatement handleUpdate(String sql, StatementType stmtType, builder.append('`').append(table).append('`').append(" UPDATE"); // .append(sql.substring(index + 3)); addMutationSetting(sql, builder, positions, settings, index + 3); return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, - compressAlgorithm, compressLevel, format, file, parameters, null, settings, null); + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0); } private ClickHouseSqlStatement handleInFileForInsertQuery(String sql, StatementType stmtType, String cluster, - String database, String table, String input, String compressAlgorithm, String compressLevel, String format, - String file, List parameters, Map positions, Map settings, - Set tempTables) { + String database, String table, String input, String compressAlgorithm, String compressLevel, String format, + String file, List parameters, Map positions, Map settings, + Set tempTables, int valueGroups) { StringBuilder builder = new StringBuilder(sql.length()); builder.append(sql.substring(0, positions.get("FROM"))); Integer index = positions.get("SETTINGS"); @@ -115,7 +115,7 @@ private ClickHouseSqlStatement handleInFileForInsertQuery(String sql, StatementT builder.append("FORMAT ").append(format); } return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, - compressAlgorithm, compressLevel, format, file, parameters, null, settings, null); + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, valueGroups); } private ClickHouseSqlStatement handleOutFileForSelectQuery(String sql, StatementType stmtType, String cluster, @@ -129,14 +129,14 @@ private ClickHouseSqlStatement handleOutFileForSelectQuery(String sql, Statement builder.append(sql.substring(index)); } return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, - compressAlgorithm, compressLevel, format, file, parameters, null, settings, null); + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0); } @Override public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType, String cluster, String database, String table, String input, String compressAlgorithm, String compressLevel, String format, String file, List parameters, Map positions, Map settings, - Set tempTables) { + Set tempTables, int valueGroups) { boolean hasFile = allowLocalFile && !ClickHouseChecker.isNullOrEmpty(file) && file.charAt(0) == '\''; ClickHouseSqlStatement s = null; if (stmtType == StatementType.DELETE) { @@ -149,7 +149,7 @@ public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType format, file, parameters, positions, settings, tempTables); } else if (stmtType == StatementType.INSERT && hasFile) { s = handleInFileForInsertQuery(sql, stmtType, cluster, database, table, input, compressAlgorithm, - compressLevel, format, file, parameters, positions, settings, tempTables); + compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups); } else if (stmtType == StatementType.SELECT && hasFile) { s = handleOutFileForSelectQuery(sql, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java index 434537ac5..6f1af61a6 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java @@ -52,7 +52,7 @@ public String handleParameter(String cluster, String database, String table, int public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType, String cluster, String database, String table, String input, String compressAlgorithm, String compressLevel, String format, String file, List parameters, Map positions, Map settings, - Set tempTables) { + Set tempTables, int valueGroup) { return null; } } diff --git a/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj b/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj index 87448c13d..e1b32b304 100644 --- a/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj +++ b/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj @@ -174,6 +174,7 @@ TOKEN_MGR_DECLS: { String compressLevel = null; String format = null; String file = null; + int valueGroups = 0; final List parameters = new ArrayList<>(); final Map positions = new HashMap<>(); @@ -276,12 +277,12 @@ TOKEN_MGR_DECLS: { if (handler != null) { s = handler.handleStatement( - sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables); + sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups); } if (s == null) { s = new ClickHouseSqlStatement( - sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables); + sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups); } // reset variables @@ -323,6 +324,9 @@ TOKEN_MGR_DECLS: { this.settings.put(key.toLowerCase(Locale.ROOT), value); } + void incValueGroup() { + this.valueGroups++; + } } SKIP: { @@ -366,7 +370,7 @@ SKIP: { } } } - | { append(image); } + | { append(image); } | "/*" { commentNestingDepth = 1; append(image); }: MULTI_LINE_COMMENT } @@ -418,6 +422,7 @@ void stmt(): {} { | optimizeStmt() { token_source.stmtType = StatementType.OPTIMIZE; } | renameStmt() { token_source.stmtType = StatementType.RENAME; } | revokeStmt() { token_source.stmtType = StatementType.REVOKE; } + | selectStmt() { token_source.stmtType = StatementType.SELECT; } | selectStmt() { token_source.stmtType = StatementType.SELECT; } | setStmt() { token_source.stmtType = StatementType.SET; } | showStmt() { token_source.stmtType = StatementType.SHOW; } @@ -569,12 +574,12 @@ void dataClause(): {} { try { LOOKAHEAD(2) { token_source.addPosition(token); } { token_source.addCustomKeywordPosition(ClickHouseSqlStatement.KEYWORD_VALUES_START, token); } - columnExprList() + columnExprList() { token_source.incValueGroup(); } { token_source.addCustomKeywordPosition(ClickHouseSqlStatement.KEYWORD_VALUES_END, token); } ( LOOKAHEAD(2) ()? - { token_source.removePosition(ClickHouseSqlStatement.KEYWORD_VALUES_START); } + { token_source.removePosition(ClickHouseSqlStatement.KEYWORD_VALUES_START); token_source.incValueGroup(); } columnExprList() { token_source.removePosition(ClickHouseSqlStatement.KEYWORD_VALUES_END); } )* @@ -656,6 +661,7 @@ void showStmt(): {} { LOOKAHEAD(2) ( databaseIdentifier(true)) + | LOOKAHEAD(2) (LOOKAHEAD(1) )? (LOOKAHEAD(1) )? anyIdentifier() | LOOKAHEAD(2) ( tableIdentifier(true)) | LOOKAHEAD(2) ((LOOKAHEAD(2) )? (LOOKAHEAD(2) )? tableIdentifier(true)) ) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index e3f65d6be..593a060d2 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -125,9 +125,14 @@ public static Object[][] testPreparedStatementInsertSQLDP() { + "#! line comment3 ?\n" + "/* block comment ? \n */" + "INSERT INTO `with_complex_id`(`v?``1`, \"v?\"\"2\",`v?\\`3`, \"v?\\\"4\") VALUES (?, ?, ?, ?);", 1, false, 4}, + {"-- line comment1 ?\n" +// + "# line comment2 ?\n" +// + "#! line comment3 ?\n" + + "/* block comment ? \n */" + + "INSERT INTO `with_complex_id`(`v?``1`, \"v?\"\"2\",`v?\\`3`, \"v?\\\"4\") VALUES (?, ?, ?, ?);", 1, false, 4}, { "INSERT INTO `test_stmt_split2` VALUES (1, 'abc'), (2, '?'), (3, '?')", 3, false, 0 }, { "INSERT INTO `with_complex_id`(`v?``1`, \"v?\"\"2\",`v?\\`3`, \"v?\\\"4\") VALUES (?, ?, ?, ?);", 1, false, 4}, - { "INSERT INTO tt SELECT now(), 10, 20.0, 30", -1, true, 0 }, + { "INSERT INTO tt SELECT now(), 10, 20.0, 30", 0, true, 0 }, { "INSERT INTO `users` (`name`, `last_login`, `password`, `id`) VALUES\n" + " (?, `parseDateTimeBestEffort`(?, ?), ?, 1)\n", 1, false, 4 }, }; diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JavaCCParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JavaCCParserTest.java index 86fcd334a..838e894c4 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JavaCCParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JavaCCParserTest.java @@ -3,7 +3,7 @@ import org.testng.annotations.Ignore; -@Ignore +//@Ignore public class JavaCCParserTest extends BaseSqlParserFacadeTest { public JavaCCParserTest() throws Exception { super(SqlParserFacade.SQLParser.JAVACC.name()); From f02382fa9aba91d067c47f96af5143729a586a41 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 24 Oct 2025 12:13:18 -0700 Subject: [PATCH 21/30] Fixed some keyword issues --- .../internal/parser/antlr4/ClickHouseLexer.g4 | 2 ++ .../internal/parser/antlr4/ClickHouseParser.g4 | 9 +++++++++ .../jdbc/internal/SqlParserFacade.java | 4 ++-- .../jdbc/internal/BaseSqlParserFacadeTest.java | 16 +++++++++++++++- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 index c8ec160a3..3b13af550 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 @@ -377,6 +377,8 @@ WORKLOAD : W O R K L O A D; WRITABLE : W R I T A B L E; YEAR : Y E A R | Y Y Y Y; ZKPATH : Z K P A T H; +SUM : S U M; +AVG : A V G; JSON_FALSE : 'false'; JSON_TRUE : 'true'; diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 index 468b280a9..b87541c42 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 @@ -513,6 +513,7 @@ fromClause | FROM identifier LPAREN JDBC_PARAM_PLACEHOLDER RPAREN | FROM selectStmt | FROM identifier LPAREN viewParam (COMMA viewParam)? RPAREN + | FROM tableFunctionExpr ; viewParam @@ -1315,6 +1316,7 @@ keyword | LIVE | LOCAL | LOGS + | LOG | MATERIALIZE | MATERIALIZED | MAX @@ -1345,6 +1347,7 @@ keyword | RANGE | RELOAD | REMOVE + | REMOTE | RENAME | REPLACE | REPLICA @@ -1391,7 +1394,9 @@ keyword | USE | USING | USER + | USERS | UUID + | URL | VALUES | VIEW | VOLUME @@ -1401,6 +1406,10 @@ keyword | WINDOW | WITH | QUERIES + | SUM + | AVG + | REFRESH + | EXPLAIN ; keywordForAlias diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java index a3a49ecb0..7b6cab0c2 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java @@ -44,7 +44,7 @@ public ParsedStatement parsedStatement(String sql) { } // TODO: set roles stmt.setInsert(parsedStmt.getStatementType() == StatementType.INSERT); - stmt.setHasErrors(false); + stmt.setHasErrors(parsedStmt.getStatementType() == StatementType.UNKNOWN); stmt.setHasResultSet(isStmtWithResultSet(parsedStmt)); return stmt; } @@ -64,7 +64,7 @@ public ParsedPreparedStatement parsePreparedStatement(String sql) { stmt.setUseDatabase(parsedStmt.getDatabase()); } stmt.setInsert(parsedStmt.getStatementType() == StatementType.INSERT); - stmt.setHasErrors(false); + stmt.setHasErrors(parsedStmt.getStatementType() == StatementType.UNKNOWN); stmt.setHasResultSet(isStmtWithResultSet(parsedStmt)); stmt.setTable(parsedStmt.getTable()); stmt.setInsertWithSelect(parsedStmt.containsKeyword("SELECT") && (parsedStmt.getStatementType() == StatementType.INSERT)); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index 593a060d2..c2cab50c5 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -239,14 +239,28 @@ public static Object[][] testCTEStmtsDP() { public void testMiscStatements(String sql, int args) { ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertEquals(stmt.getArgCount(), args); - Assert.assertFalse(stmt.isHasErrors()); + Assert.assertFalse(stmt.isHasErrors(), "Statement has errors"); } @DataProvider public Object[][] testMiscStmtDp() { return new Object[][] { + {"SELECT x, a FROM (SELECT arrayJoin(['Hello', 'Goodbye']) AS x, [1, 2, 3] AS arr) ARRAY JOIN arr AS a", 0}, + {"SELECT quantilesTiming(0.1, 0.5, 0.9)(dummy) FROM remote('127.0.0.{2,3}', 'system', 'one') GROUP BY 1 WITH TOTALS", 0}, // FROM remote issue + {"SELECT StartDate, sumMerge(Visits) AS Visits, uniqMerge(Users) AS Users FROM basic_00040 GROUP BY StartDate ORDER BY StartDate", 0}, // keywords + {"SELECT uniq(URL) FROM test.hits WHERE TraficSourceID IN (7)", 0}, // keywords URL {"SELECT INTERVAL '1 day'", 0}, {"SELECT INTERVAL 1 day", 0}, + {"SET extremes = 1", 0}, + {"CREATE TABLE check_query_log (N UInt32,S String) Engine = Log", 0 }, + {"CREATE TABLE log (x UInt8) ENGINE = StripeLog", 0}, + {"CREATE TABLE check_query_log (N UInt32,S String) Engine = MergeTree", 0 }, + {"CREATE TABLE check_query_log (N UInt32,S String) Engine = ReplacingMergeTree", 0 }, + {"select abs(log(e()) - 1) < 1e-8", 0}, + {"SELECT SearchEngineID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) " + + " FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY SearchEngineID, ClientIP " + + " ORDER BY c DESC LIMIT 10", 0}, + {"SELECT (id % 10) AS key, count() FROM 03279_test_database.test_table_1 GROUP BY key ORDER BY key", 0}, {"SELECT ?", 1}, {"(SELECT ?)", 1}, {"SELECT * FROM table key WHERE ts = ?", 1}, From 6372e4ed995c4478fde7492bab571c306c0757bc Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 24 Oct 2025 21:58:26 -0700 Subject: [PATCH 22/30] Fixed some more antlr4 issues --- .../clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 | 4 +++- .../jdbc/internal/parser/antlr4/ClickHouseParser.g4 | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 index 3b13af550..1b77a9d85 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseLexer.g4 @@ -385,7 +385,9 @@ JSON_TRUE : 'true'; // Tokens -IDENTIFIER: (LETTER | UNDERSCORE) (LETTER | UNDERSCORE | DEC_DIGIT)* +IDENTIFIER: + (LETTER | UNDERSCORE) (LETTER | UNDERSCORE | DEC_DIGIT)* + | DEC_DIGIT+ (LETTER | UNDERSCORE) (LETTER | UNDERSCORE | DEC_DIGIT)* | BACKQUOTE ( ~([\\`]) | (BACKSLASH .) | (BACKQUOTE BACKQUOTE))* BACKQUOTE | QUOTE_DOUBLE (~([\\"]) | (BACKSLASH .) | (QUOTE_DOUBLE QUOTE_DOUBLE))* QUOTE_DOUBLE ; diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 index b87541c42..65fc84967 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 @@ -510,6 +510,7 @@ topClause fromClause : FROM joinExpr + | FROM tableIdentifier | FROM identifier LPAREN JDBC_PARAM_PLACEHOLDER RPAREN | FROM selectStmt | FROM identifier LPAREN viewParam (COMMA viewParam)? RPAREN @@ -661,7 +662,7 @@ exchangeStmt // SET statement setStmt - : SET settingExprList + : SET IDENTIFIER EQ_SINGLE literal ; // SET ROLE statement From 4f45dc8b386132a3710881880ccae5442f4e5f3e Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 27 Oct 2025 13:23:24 -0700 Subject: [PATCH 23/30] Added a few SQL tests fro Statement/PreparedStatement. Addded more tests for parsers --- .../com/clickhouse/jdbc/DriverProperties.java | 2 +- .../jdbc/{SQLTests.java => BaseSQLTests.java} | 19 ++-- .../jdbc/PreparedStatementSQLTest.java | 45 +++++++++ .../com/clickhouse/jdbc/StatementSQLTest.java | 50 ++++++++++ .../internal/BaseSqlParserFacadeTest.java | 10 ++ .../resources/PreparedStatementSQLTests.yaml | 34 +++++++ .../src/test/resources/StatementSQLTests.yaml | 91 +++++++++++++++++++ jdbc-v2/src/test/resources/datasets.yaml | 18 +++- 8 files changed, 259 insertions(+), 10 deletions(-) rename jdbc-v2/src/test/java/com/clickhouse/jdbc/{SQLTests.java => BaseSQLTests.java} (95%) create mode 100644 jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementSQLTest.java create mode 100644 jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementSQLTest.java create mode 100644 jdbc-v2/src/test/resources/PreparedStatementSQLTests.yaml create mode 100644 jdbc-v2/src/test/resources/StatementSQLTests.yaml diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java index 65b737d43..207c7ef89 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java @@ -66,7 +66,7 @@ public enum DriverProperties { *
  • JAVACC - parser extracts required information but PreparedStatement parameters parsed separately.
    • * */ - SQL_PARSER("jdbc_sql_parser", "ANTLR4", List.of("ANTLR4", "ANTLR4_PARAMS_PARSER", "JAVACC")), + SQL_PARSER("jdbc_sql_parser", "JAVACC", List.of("ANTLR4", "ANTLR4_PARAMS_PARSER", "JAVACC")), ; diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/SQLTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/BaseSQLTests.java similarity index 95% rename from jdbc-v2/src/test/java/com/clickhouse/jdbc/SQLTests.java rename to jdbc-v2/src/test/java/com/clickhouse/jdbc/BaseSQLTests.java index 0177568c5..580a12995 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/SQLTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/BaseSQLTests.java @@ -29,12 +29,12 @@ import static org.testng.Assert.fail; @Test(groups = {"integration"}) -public class SQLTests extends JdbcIntegrationTest { +public class BaseSQLTests extends JdbcIntegrationTest { private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - @Test(groups = {"integration"}, dataProvider = "testSQLQueryWithResultSetDP") + @Test(groups = {"integration"}, dataProvider = "testSQLQueryWithResultSetDP", enabled = false) public void testSQLQueryWithResultSet(Map tables, SQLTestCase testCase) throws Exception { try (Connection connection = getJdbcConnection()) { @@ -61,9 +61,13 @@ public void testSQLQueryWithResultSet(Map tables, SQLTestCa @DataProvider(name = "testSQLQueryWithResultSetDP") public static Object[][] testSQLQueryWithResultSetDP() throws Exception { - ClassLoader classLoader = SQLTests.class.getClassLoader(); - try (InputStream datasetsInput = classLoader.getResourceAsStream("datasets.yaml"); - InputStream tests = classLoader.getResourceAsStream("SQLTests.yaml")) { + return loadTestData("datasets.yaml", "SQLTests.yaml"); + } + + protected static Object[][] loadTestData(String datasetsSource, String sqlTestsSource) throws Exception { + ClassLoader classLoader = BaseSQLTests.class.getClassLoader(); + try (InputStream datasetsInput = classLoader.getResourceAsStream(datasetsSource); + InputStream tests = classLoader.getResourceAsStream(sqlTestsSource)) { // Parse resource files TestDataset[] datasets = yamlMapper.readValue(datasetsInput, TestDataset[].class); @@ -91,12 +95,11 @@ public static Object[][] testSQLQueryWithResultSetDP() throws Exception { return testData; } catch (Exception e) { - e.printStackTrace(); throw new RuntimeException(e); } } - private void setupTables(Map tables, Connection connection) throws Exception { + protected void setupTables(Map tables, Connection connection) throws Exception { for (Map.Entry entry : tables.entrySet()) { String tableName = entry.getKey(); TestDataset dataset = entry.getValue(); @@ -155,7 +158,7 @@ public int rsMetadataChecks(ResultSet rs, SQLTestCase testCase, Map tables) throws Exception { + protected int dataCheck(ResultSet rs, SQLTestCase testCase, Map tables) throws Exception { List> checks = testCase.getChecks().stream().filter(cm -> DATA_CHECKS.containsKey(cm.getName())) .map(cm -> Pair.of(cm, DATA_CHECKS.get(cm.getName()))).collect(Collectors.toList()); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementSQLTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementSQLTest.java new file mode 100644 index 000000000..5eaa1c88f --- /dev/null +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementSQLTest.java @@ -0,0 +1,45 @@ +package com.clickhouse.jdbc; + + +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Map; + +/** + * Integration test for prepared statement. Testing SQL with prepared statement is main focus of this test. + * Any tests that relate to schema, data type, tricky SQL comes here. + * + */ +@Test(groups = {"integration"}, enabled = false) +public class PreparedStatementSQLTest extends BaseSQLTests { + + @Test(groups = {"integration"}, dataProvider = "testSQLStatements") + public void testQuery(Map tables, SQLTestCase testCase) throws Exception { + + try (Connection connection = getJdbcConnection()) { + setupTables(tables, connection); + + try (PreparedStatement stmt = connection.prepareStatement(testCase.getQuery()); + ResultSet rs = stmt.executeQuery()) { + + int checkCount = 0; + checkCount += rsMetadataChecks(rs, testCase, tables); + checkCount += dataCheck(rs, testCase, tables); + Assert.assertEquals(checkCount, testCase.getChecks().size(), "Check count does not match"); + Assert.assertTrue(checkCount > 0, "Test without checks"); + } + } + } + + + @DataProvider(name = "testSQLStatements") + public static Object[][] testSQLStatementsDP() throws Exception { + return loadTestData("datasets.yaml", "PreparedStatementSQLTests.yaml"); + } + +} diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementSQLTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementSQLTest.java new file mode 100644 index 000000000..bbb86ff7c --- /dev/null +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementSQLTest.java @@ -0,0 +1,50 @@ +package com.clickhouse.jdbc; + + +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Integration test for prepared statement. Testing SQL with prepared statement is main focus of this test. + * Any tests that relate to schema, data type, tricky SQL comes here. + * + */ +@Test(groups = {"integration"}) +public class StatementSQLTest extends BaseSQLTests { + + private AtomicBoolean isTablesSetup = new AtomicBoolean(false); + + + @Test(groups = {"integration"}, dataProvider = "testSQLStatements") + public void testQuery(Map tables, SQLTestCase testCase) throws Exception { + + try (Connection connection = getJdbcConnection()) { + setupTables(tables, connection); + + try (Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(testCase.getQuery())) { + + int checkCount = 0; + checkCount += rsMetadataChecks(rs, testCase, tables); + checkCount += dataCheck(rs, testCase, tables); + Assert.assertEquals(checkCount, testCase.getChecks().size(), "Check count does not match"); + Assert.assertTrue(checkCount > 0, "Test without checks"); + } + } + } + + + @DataProvider(name = "testSQLStatements") + public static Object[][] testSQLStatementsDP() throws Exception { + return loadTestData("datasets.yaml", "StatementSQLTests.yaml"); + } + +} diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index c2cab50c5..7740f10ad 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -344,9 +344,16 @@ public Object[][] testMiscStmtDp() { {" \n INSERT INTO TESTING \n SELECT ? AS num", 1}, {" SELECT '##?0.1' as f, ? as a\n #this is debug \n FROM table", 1}, {"WITH '#!?0.1' as f, ? as a\n #this is debug \n SELECT * FROM a", 1}, + {SELECT_WITH_WHERE_CLAUSE_FUNC_WITH_PARAMS, 2} }; } + private static final String SELECT_WITH_WHERE_CLAUSE_FUNC_WITH_PARAMS = "SELECT `source`.`id` AS `id`,\n" + + " `source`.`val` AS `val` FROM\n" + + " (with base as (\\n select 1 id, 'abc' val\\n)\\nselect * from base)\n" + + " AS `source`\n" + + " WHERE `positionCaseInsensitiveUTF8`(`source`.`val`, ?) > ? LIMIT 2000"; + private static final String INSERT_WITH_COMMENTS = "-- line comment1 ?\n" + "# line comment2 ?\n" + "#! line comment3 ?\n" @@ -459,6 +466,7 @@ public static Object[][] testStatementWithoutResultSetDP() { return new Object[][]{ /* has result set */ {"SELECT * FROM test_table", 0, true}, + {"SELECT 1 table WHERE 1 = ?", 1, true}, {"SHOW CREATE TABLE `db`.`test_table`", 0, true}, {"SHOW CREATE TEMPORARY TABLE `db1`.`tmp_table`", 0, true}, {"SHOW CREATE DICTIONARY dict1", 0, true}, @@ -559,6 +567,7 @@ public static Object[][] testStatementWithoutResultSetDP() { {"CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin", 0, false}, {"CREATE NAMED COLLECTION foobar AS a = '1', b = '2' OVERRIDABLE", 0, false}, {"alter table t2 alter column v type Int32", 0, false}, + {"alter table t alter column j default 1", 0, false}, {"ALTER TABLE t MODIFY COLUMN j default 1", 0, false}, {"ALTER TABLE t MODIFY COMMENT 'comment'", 0, false}, {"ALTER TABLE t ADD COLUMN id Int32 AFTER v", 0, false}, @@ -567,6 +576,7 @@ public static Object[][] testStatementWithoutResultSetDP() { {"DELETE FROM table WHERE a = ?", 1, false}, {"DELETE FROM table WHERE a = ? AND b = ?", 2, false}, {"DELETE FROM hits WHERE Title LIKE '%hello%';", 0, false}, + {"DELETE FROM t WHERE true", 0, false}, {"SYSTEM START FETCHES", 0, false}, {"SYSTEM RELOAD DICTIONARIES", 0, false}, {"SYSTEM RELOAD DICTIONARIES ON CLUSTER `default`", 0, false}, diff --git a/jdbc-v2/src/test/resources/PreparedStatementSQLTests.yaml b/jdbc-v2/src/test/resources/PreparedStatementSQLTests.yaml new file mode 100644 index 000000000..ba680e8fb --- /dev/null +++ b/jdbc-v2/src/test/resources/PreparedStatementSQLTests.yaml @@ -0,0 +1,34 @@ +--- +- name: select_all_events + query: SELECT * FROM events + tables: + events: datasets/events +# checks: +# - name: row_count +# expected: ${events.rowCount} +# - name: column_count +# expected: ${events.columnsCount} +# - name: column_names +# expected: ${events.columnNames} +# - name: column_types +# expected: ${events.columnTypes} +- name: cte_select_with_params_01 + query: | + WITH + toDate('2025-08-20') as DATE_END + ,events AS ( + SELECT 1 + ) + SELECT * + FROM events + tables: + events: datasets/events + checks: + - name: row_count + expected: 1 + - name: column_count + expected: 1 + - name: column_names + expected: ["1"] + - name: column_types + expected: ["UInt8"] \ No newline at end of file diff --git a/jdbc-v2/src/test/resources/StatementSQLTests.yaml b/jdbc-v2/src/test/resources/StatementSQLTests.yaml new file mode 100644 index 000000000..579899421 --- /dev/null +++ b/jdbc-v2/src/test/resources/StatementSQLTests.yaml @@ -0,0 +1,91 @@ +--- +- name: select_all_events + query: SELECT * FROM events + tables: + events: datasets/events +# checks: +# - name: row_count +# expected: ${events.rowCount} +# - name: column_count +# expected: ${events.columnsCount} +# - name: column_names +# expected: ${events.columnNames} +# - name: column_types +# expected: ${events.columnTypes} +- name: select_with_01 + query: | + WITH + toDate('2025-08-20') as DATE_END, + cte_events AS ( + SELECT 1, DATE_END + ) + SELECT *, DATE_END + FROM cte_events + tables: + events: datasets/events + checks: + - name: column_count + expected: 3 + - name: column_names + expected: ["1", "cte_events.DATE_END", "DATE_END"] + - name: column_types + expected: ["UInt8", "Date", "Date"] +- name: select_with_02 + query: | + WITH + toDate('2025-08-20') as DATE_END, + cte_events AS ( + SELECT 1 + ) + SELECT * + FROM cte_events + tables: + events: datasets/events + checks: + - name: row_count + expected: 1 + - name: column_count + expected: 1 + - name: column_names + expected: ["1"] + - name: column_types + expected: ["UInt8"] +- name: select_with_fill_03 + query: | + SELECT + value, + toUnixTimestamp(ts1) * 1000 AS timestamp + FROM + ( + SELECT + count() AS value, + toStartOfInterval(ts, toIntervalDay(1)) AS ts1 + FROM events + WHERE (ts >= toDateTime(1736035200)) AND (ts <= toDateTime(1751587199)) + GROUP BY ts1 + ORDER BY ts1 ASC WITH FILL FROM toDateTime(1736035200) TO toDateTime(1751587199) STEP toIntervalDay(1) + ) + tables: + events: datasets/events + checks: + - name: row_count + expected: 180 + - name: column_count + expected: 2 + - name: column_names + expected: ["value", "timestamp"] + - name: column_types + expected: ["UInt64", "UInt64"] +- name: explain_stmt_01 + query: EXPLAIN SELECT 1 + tables: + events: datasets/empty_table + checks: + - name: row_count + expected: 2 + - name: column_count + expected: 1 + - name: column_names + expected: ["explain"] + - name: column_types + expected: ["String"] \ No newline at end of file diff --git a/jdbc-v2/src/test/resources/datasets.yaml b/jdbc-v2/src/test/resources/datasets.yaml index 5b16b8582..fc4c27a4e 100644 --- a/jdbc-v2/src/test/resources/datasets.yaml +++ b/jdbc-v2/src/test/resources/datasets.yaml @@ -12,4 +12,20 @@ - ["4", "event2", "2025-01-02 00:02:00.004000000", "30.2"] - ["5", "event1", "2025-01-05 00:01:10.002000000", "12.8"] - ["6", "event3", "2025-01-02 00:01:20.003000000", "45.9"] - - ["7", "event4", "2025-01-02 00:01:30.004000000", "22.1"] \ No newline at end of file + - ["7", "event4", "2025-01-02 00:01:30.004000000", "22.1"] +- name: empty_table + columns: + - "Int8 Int8" + - "Int16 Int16" + - "Int32 Int32" + - "Int64 Int64" + - "UInt8 UInt8" + - "UInt16 UInt16" + - "UInt32 UInt32" + - "UInt64 UInt64" + - "Float32 Float32" + - "Float64 Float64" + - "String String" + - "DateTime DateTime" + - "DateTime64 DateTime64" + data: [] \ No newline at end of file From b930ca2b63c526bd5fc488684d48cc133fdc87f8 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 27 Oct 2025 13:56:40 -0700 Subject: [PATCH 24/30] Fixed javaCC for set roles statements and getting correct table name --- .../com/clickhouse/jdbc/DriverProperties.java | 3 ++- .../jdbc/internal/SqlParserFacade.java | 26 +++++++++++++++++-- .../parser/javacc/ClickHouseSqlStatement.java | 3 +++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java index 207c7ef89..f623edda4 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java @@ -66,7 +66,8 @@ public enum DriverProperties { *
  • JAVACC - parser extracts required information but PreparedStatement parameters parsed separately.
    • * */ - SQL_PARSER("jdbc_sql_parser", "JAVACC", List.of("ANTLR4", "ANTLR4_PARAMS_PARSER", "JAVACC")), +// SQL_PARSER("jdbc_sql_parser", "JAVACC", List.of("ANTLR4", "ANTLR4_PARAMS_PARSER", "JAVACC")), + SQL_PARSER("jdbc_sql_parser", "ANTLR4", List.of("ANTLR4", "ANTLR4_PARAMS_PARSER", "JAVACC")), ; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java index 7b6cab0c2..27e5d83d8 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java @@ -42,7 +42,25 @@ public ParsedStatement parsedStatement(String sql) { if (parsedStmt.getStatementType() == StatementType.USE) { stmt.setUseDatabase(parsedStmt.getDatabase()); } - // TODO: set roles + + String rolesCount = parsedStmt.getSettings().get("_ROLES_COUNT"); + if (rolesCount != null) { + int rolesCountInt = Integer.parseInt(rolesCount); + ArrayList roles = new ArrayList<>(rolesCountInt); + boolean resetRoles = false; + for (int i = 0; i < rolesCountInt; i++) { + String role = parsedStmt.getSettings().get("_ROLE_" + i); + if (role.equalsIgnoreCase("NONE")) { + resetRoles = true; + } + roles.add(parsedStmt.getSettings().get("_ROLE_" + i)); + } + if (resetRoles) { + roles.clear(); + } + stmt.setRoles(roles); + } + stmt.setInsert(parsedStmt.getStatementType() == StatementType.INSERT); stmt.setHasErrors(parsedStmt.getStatementType() == StatementType.UNKNOWN); stmt.setHasResultSet(isStmtWithResultSet(parsedStmt)); @@ -66,7 +84,11 @@ public ParsedPreparedStatement parsePreparedStatement(String sql) { stmt.setInsert(parsedStmt.getStatementType() == StatementType.INSERT); stmt.setHasErrors(parsedStmt.getStatementType() == StatementType.UNKNOWN); stmt.setHasResultSet(isStmtWithResultSet(parsedStmt)); - stmt.setTable(parsedStmt.getTable()); + String tableName = parsedStmt.getTable(); + if (parsedStmt.getDatabase() != null && parsedStmt.getTable() != null) { + tableName = String.format("%s.%s", parsedStmt.getDatabase(), parsedStmt.getTable()); + } + stmt.setTable(tableName); stmt.setInsertWithSelect(parsedStmt.containsKeyword("SELECT") && (parsedStmt.getStatementType() == StatementType.INSERT)); stmt.setAssignValuesGroups(parsedStmt.getValueGroups()); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java index 5afad67e7..7dfd22876 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java @@ -27,6 +27,9 @@ public class ClickHouseSqlStatement { public static final String KEYWORD_VALUES_START = "ValuesStart"; public static final String KEYWORD_VALUES_END = "ValuesEnd"; + public static final String ROLES_COUNT_SETTINGS_KEY = "_ROLES_COUNT"; + public static final String ROLES_PREFIX_SETTINGS_KEY = "_ROLE_"; + private final String sql; private final StatementType stmtType; private final String cluster; From 24c8e8199a3c0f442f2369b8192b208eb0269e9c Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 27 Oct 2025 16:14:42 -0700 Subject: [PATCH 25/30] Added new statements to javacc. fixed some existing statements --- .../internal/parser/javacc/StatementType.java | 6 ++- .../src/main/javacc/ClickHouseSqlParser.jj | 44 +++++++++++++------ .../internal/BaseSqlParserFacadeTest.java | 5 ++- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/StatementType.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/StatementType.java index 03c47c884..ecf7af12d 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/StatementType.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/StatementType.java @@ -28,8 +28,12 @@ public enum StatementType { UPDATE(LanguageType.DML, OperationType.WRITE, false), // the upcoming light-weight update statement USE(LanguageType.DDL, OperationType.UNKNOWN, true), // use statement WATCH(LanguageType.DDL, OperationType.UNKNOWN, true), // watch statement - TRANSACTION(LanguageType.TCL, OperationType.WRITE, true); // TCL statement + TRANSACTION(LanguageType.TCL, OperationType.WRITE, true), // TCL statement + UNDROP(LanguageType.DDL, OperationType.UNKNOWN, false), + MOVE(LanguageType.DCL, OperationType.UNKNOWN, false), + EXCHANGE(LanguageType.DML, OperationType.UNKNOWN, false), + ; private LanguageType langType; private OperationType opType; private boolean idempotent; diff --git a/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj b/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj index e1b32b304..fae8e3cdd 100644 --- a/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj +++ b/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj @@ -415,6 +415,7 @@ void stmt(): {} { | detachStmt() { token_source.stmtType = StatementType.DETACH; } | dropStmt() { token_source.stmtType = StatementType.DROP; } | existsStmt() { token_source.stmtType = StatementType.EXISTS; } + | exchangeStmt() { token_source.stmtType = StatementType.EXCHANGE; } | explainStmt() { token_source.stmtType = StatementType.EXPLAIN; } | insertStmt() { token_source.stmtType = StatementType.INSERT; } | grantStmt() { token_source.stmtType = StatementType.GRANT; } @@ -428,10 +429,12 @@ void stmt(): {} { | showStmt() { token_source.stmtType = StatementType.SHOW; } | systemStmt() { token_source.stmtType = StatementType.SYSTEM; } | truncateStmt() { token_source.stmtType = StatementType.TRUNCATE; } + | undropStmt() { token_source.stmtType = StatementType.UNDROP; } | updateStmt() { token_source.stmtType = StatementType.UPDATE; } | useStmt() { token_source.stmtType = StatementType.USE; } | watchStmt() { token_source.stmtType = StatementType.WATCH; } | txStmt() { token_source.stmtType = StatementType.TRANSACTION; } + | moveStmt() { token_source.stmtType = StatementType.MOVE; } } // https://clickhouse.tech/docs/en/sql-reference/statements/alter/ @@ -476,9 +479,21 @@ void checkStmt(): {} { // not interested anyExprList() } +void undropStmt(): {} { // not interested + anyExprList() +} + +void moveStmt(): {} { // not interested + (LOOKAHEAD(2) | )? anyExprList() +} + +void exchangeStmt(): {} { // not interested + ( | ) anyExprList() +} + // https://clickhouse.tech/docs/en/sql-reference/statements/create/ void createStmt(): {} { - ( + (LOOKAHEAD(2) )? ( LOOKAHEAD(2) ( { token_source.addPosition(token); } @@ -617,7 +632,6 @@ void revokeStmt(): {} { // not interested // https://clickhouse.tech/docs/en/sql-reference/statements/select/ void selectStmt(): {} { - // FIXME with (select 1), (select 2), 3 select * (withClause())?
    )? (LOOKAHEAD(2) )? - tableIdentifier(true) (clusterClause())? + (LOOKAHEAD(2) )? (LOOKAHEAD(2)
    )? (LOOKAHEAD(2) )? anyExprList() } // upcoming lightweight mutation - see https://github.com/ClickHouse/ClickHouse/issues/19627 void updateStmt(): {} { - { token_source.addPosition(token); } tableIdentifier(true) + { token_source.addPosition(token); } tableIdentifier(true) (LOOKAHEAD(2) clusterClause())? { token_source.addPosition(token); } anyExprList() } @@ -795,6 +808,7 @@ void columnExpr(): { Token t; } { | (LOOKAHEAD(2) anyExprList())? | (LOOKAHEAD(2) anyExprList())? | anyExprList() + | LOOKAHEAD(3) literal() literal() | (LOOKAHEAD(2) macro())+ | LOOKAHEAD(2, { !(tokenIn(1, INF, NAN, NULL) && tokenIn(2, DOT)) }) literal() | LOOKAHEAD(2, { getToken(2).kind == LPAREN }) functionExpr() @@ -1004,9 +1018,9 @@ Token anyKeyword(): { Token t; } { ( // leading keywords(except with) t = | t = | t = | t = | t = | t = | t = - | t = | t = | t = | t = | t = | t = | t = - | t = | t = | t = | t = | t = | t = | t = + | t = | t = | t = | t = | t = // others | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = @@ -1017,7 +1031,7 @@ Token anyKeyword(): { Token t; } { | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t =
    | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = - | t = | t = | t = + | t = | t = | t = | t = // interval | t = | t = | t = | t = | t = | t = | t = | t = // values @@ -1030,16 +1044,16 @@ Token keyword(): { Token t; } { ( // leading keywords(except with) t = | t = | t = | t = | t = | t = | t = - | t = | t = | t = | t = | t = | t = | t = - | t = | t = | t = | t = | t = | t = | t = + | t = | t = | t = | t = | t = // others | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t = | t =
    | t = | t = | t = | t = | t = | t = - | t = | t = | t = | t = + | t = | t = | t = | t = | t = // interval | t = | t = | t = | t = | t = | t = | t = | t = // values @@ -1060,10 +1074,12 @@ TOKEN: { | > |

    > | > + | > |

    > | > | > | > + | > |

    > | > | > @@ -1078,6 +1094,7 @@ TOKEN: { | > | > | > + |

    > | > | > @@ -1156,6 +1173,7 @@ TOKEN: { | > | > | > + |

    > | > | > diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index 7740f10ad..9357325ce 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -348,9 +348,9 @@ public Object[][] testMiscStmtDp() { }; } - private static final String SELECT_WITH_WHERE_CLAUSE_FUNC_WITH_PARAMS = "SELECT `source`.`id` AS `id`,\n" + + private static final String SELECT_WITH_WHERE_CLAUSE_FUNC_WITH_PARAMS = "SELECT `source`.`id` AS `id`, \n" + " `source`.`val` AS `val` FROM\n" + - " (with base as (\\n select 1 id, 'abc' val\\n)\\nselect * from base)\n" + + " (with base as (\n select 1 id, 'abc' val\n) \nselect * from base)\n" + " AS `source`\n" + " WHERE `positionCaseInsensitiveUTF8`(`source`.`val`, ?) > ? LIMIT 2000"; @@ -646,6 +646,7 @@ public static Object[][] testStatementWithoutResultSetDP() { {"TRUNCATE TABLE `db1`.`table1` ON CLUSTER `default` SYNC", 0, false}, {"TRUNCATE TABLE `db1`.`table1` ON CLUSTER `default`", 0, false}, {"TRUNCATE TABLE `db1`.`table1`", 0, false}, + {"TRUNCATE TEMPORARY TABLE t", 0, false}, {"TRUNCATE DATABASE IF EXISTS db ON CLUSTER `cluster`", 0, false}, {"TRUNCATE DATABASE IF EXISTS db", 0, false}, {"TRUNCATE DATABASE `db`", 0, false}, From 5f172d2edd3f55bf22e4ce3e6a7445dc1af3f432 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 27 Oct 2025 16:24:05 -0700 Subject: [PATCH 26/30] fixed alter table - type is not required. --- .../jdbc/internal/parser/antlr4/ClickHouseParser.g4 | 4 ++-- .../com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 index 65fc84967..8f294f8fd 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 @@ -84,7 +84,7 @@ alterTableClause | MODIFY COLUMN (IF EXISTS)? nestedIdentifier COMMENT STRING_LITERAL # AlterTableClauseModifyComment | MODIFY COLUMN (IF EXISTS)? nestedIdentifier REMOVE tableColumnPropertyType # AlterTableClauseModifyRemove | MODIFY COLUMN (IF EXISTS)? tableColumnDfnt # AlterTableClauseModify - | ALTER COLUMN (IF EXISTS)? identifier TYPE columnTypeExpr codecExpr? ttlClause? settingExprList? alterTableColumnPosition? # AlterTableClauseAlterType + | ALTER COLUMN (IF EXISTS)? identifier TYPE? columnTypeExpr codecExpr? ttlClause? settingExprList? alterTableColumnPosition? # AlterTableClauseAlterType | MODIFY ORDER BY columnExpr # AlterTableClauseModifyOrderBy | MODIFY ttlClause # AlterTableClauseModifyTTL | MODIFY COMMENT literal # AlterTableClauseModifyComment @@ -662,7 +662,7 @@ exchangeStmt // SET statement setStmt - : SET IDENTIFIER EQ_SINGLE literal + : SET (identifier | settingExpr) ; // SET ROLE statement diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index 9357325ce..f6c703649 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -636,6 +636,7 @@ public static Object[][] testStatementWithoutResultSetDP() { {"EXCHANGE DICTIONARIES dict1 AND dict2", 0, false}, {"EXCHANGE DICTIONARIES dict1 AND dict2 ON CLUSTER `default`", 0, false}, {"SET profile = 'profile-name-from-the-settings-file'", 0, false}, + {"SET setting_1 = 'some value'", 0, false}, {"SET use_some_feature_flag", 0, false}, {"SET use_some_feature_flag = 'true'", 0, false}, {"SET ROLE role1", 0, false}, From 967e7dceff63dc631f0fe8b8db02bfd9915aa05b Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 28 Oct 2025 12:13:55 -0700 Subject: [PATCH 27/30] added tests with lambda --- .../jdbc/internal/BaseSqlParserFacadeTest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index f6c703649..da48e0240 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -344,7 +344,16 @@ public Object[][] testMiscStmtDp() { {" \n INSERT INTO TESTING \n SELECT ? AS num", 1}, {" SELECT '##?0.1' as f, ? as a\n #this is debug \n FROM table", 1}, {"WITH '#!?0.1' as f, ? as a\n #this is debug \n SELECT * FROM a", 1}, - {SELECT_WITH_WHERE_CLAUSE_FUNC_WITH_PARAMS, 2} + {SELECT_WITH_WHERE_CLAUSE_FUNC_WITH_PARAMS, 2}, + {"SELECT arrayFilter(x -> x > 0, [0, 1, 2, -3])", 0}, + {"SELECT [0, 1, 2, -3] arr, arrayFilter(x -> x > 0, arr)", 0}, + {"SELECT arrayFill(x, y, z -> x > y AND x < z, [5, 3, 6, 2], [4, 7, 1, 3], [10, 2, 8, 5]) AS res", 0}, + {"SELECT arrayFilter(x -> x LIKE '%World%', ['Hello', 'abc World']) AS res", 0}, + {"SELECT arrayFilter(x -> not (x is null), ['Hello', 'abc World']) AS res", 0}, + {"SELECT arrayDistinct(arrayFilter(x -> not (x is null), " + + " arrayConcat(t.s.arr1, t.s.arr2)" + + " )" + + ")", 0}, }; } From 8b2406ee8a6decd9dae49e681665fef5b232fd66 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 28 Oct 2025 17:24:35 -0700 Subject: [PATCH 28/30] fixed ANTLR4 for expressions with IP keyword --- .../clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 | 1 + .../com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 index 8f294f8fd..bd94b69c9 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/parser/antlr4/ClickHouseParser.g4 @@ -1300,6 +1300,7 @@ keyword | INSERT | INTERVAL | INTO + | IP | IS | IS_OBJECT_ID | JOIN diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index da48e0240..71bbf9b5f 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -282,6 +282,7 @@ public Object[][] testMiscStmtDp() { {"SELECT COUNT() FROM system.databases WHERE name = ?", 1}, {"alter table user delete where reg_time = ?", 1}, {"SELECT * FROM a,b WHERE id > ?", 1}, + {"select ip from myusers where tenant=?", 1}, {"DROP USER IF EXISTS default_impersonation_user", 0}, {"DROP ROLE IF EXISTS `vkonfwxapllzkkgkqdvt`", 0}, {"CREATE ROLE `kjxrsscptauligukwgmf` ON CLUSTER '{cluster}'", 0}, @@ -550,6 +551,7 @@ public static Object[][] testStatementWithoutResultSetDP() { {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster`", 0, false}, {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster` ENGINE = Replicated('clickhouse1:9000', 'test_db')", 0, false}, {"CREATE TABLE `test_table` (id UInt64) ENGINE = MergeTree() ORDER BY id ON CLUSTER `cluster` ENGINE = Replicated('clickhouse1:9000', 'test_db') COMMENT 'for tests'", 0, false}, + {"CREATE TABLE myusers ( id UInt64, ip String, url String, tenant String) ENGINE = MergeTree() PRIMARY KEY (id)", 0, false}, {"CREATE VIEW `test_db`.`source_table` source AS ( SELECT * FROM source_a UNION SELECT * FROM source_b)", 0, false}, {"CREATE OR REPLACE VIEW `test_db`.`source_table` source AS ( SELECT * FROM source_a UNION SELECT * FROM source_b)", 0, false}, {"CREATE OR REPLACE VIEW `test_db`.`source_table` source ON CLUSTER `cluster` AS ( SELECT * FROM source_a UNION SELECT * FROM source_b)", 0, false}, From 2b108b027c9b8bb74b0c1265060f1facc5b30970 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 28 Oct 2025 17:33:27 -0700 Subject: [PATCH 29/30] fixed in statement with multiple arguments --- .../jdbc/PreparedStatementTest.java | 21 ++++++++++++++++--- .../internal/BaseSqlParserFacadeTest.java | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java index 284d7a4b6..54eab52c5 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java @@ -1375,9 +1375,9 @@ public void testCTEWithUnboundCol() throws Exception { public void testWithInClause() throws Exception { try (Connection conn = getJdbcConnection()) { - String cte = "select number from system.numbers where number in (?) limit 10"; - Long[] filter = new Long[]{2L, 4L, 6L}; - try (PreparedStatement stmt = conn.prepareStatement(cte)) { + final String q1 = "select number from system.numbers where number in (?) limit 10"; + try (PreparedStatement stmt = conn.prepareStatement(q1)) { + Long[] filter = new Long[]{2L, 4L, 6L}; stmt.setArray(1, conn.createArrayOf("Int64", filter)); ResultSet rs = stmt.executeQuery(); @@ -1387,6 +1387,21 @@ public void testWithInClause() throws Exception { } Assert.assertFalse(rs.next()); } + + final String q2 = "with t as (select arrayJoin([1, 2, 3]) as a ) select * from t where a in(?, ?)"; + try (PreparedStatement stmt = conn.prepareStatement(q2)) { + Long[] filter = new Long[]{2L, 3L}; + + stmt.setInt(1, 2); + stmt.setInt(2, 3); + ResultSet rs = stmt.executeQuery(); + + for (Long filterValue : filter) { + assertTrue(rs.next()); + assertEquals(rs.getLong(1), filterValue); + } + Assert.assertFalse(rs.next()); + } } } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java index 71bbf9b5f..ae51a8bce 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java @@ -283,6 +283,7 @@ public Object[][] testMiscStmtDp() { {"alter table user delete where reg_time = ?", 1}, {"SELECT * FROM a,b WHERE id > ?", 1}, {"select ip from myusers where tenant=?", 1}, + {"SELECT myColumn FROM myTable WHERE myColumn in (?, ?, ?)", 3}, {"DROP USER IF EXISTS default_impersonation_user", 0}, {"DROP ROLE IF EXISTS `vkonfwxapllzkkgkqdvt`", 0}, {"CREATE ROLE `kjxrsscptauligukwgmf` ON CLUSTER '{cluster}'", 0}, From b773b7eced145d9d0079c665f32441c1edaa80bc Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 29 Oct 2025 09:58:28 -0700 Subject: [PATCH 30/30] fixed problem with detecting function in insert statement --- .../com/clickhouse/jdbc/DriverProperties.java | 3 +-- .../jdbc/internal/SqlParserFacade.java | 2 +- .../parser/javacc/ClickHouseSqlStatement.java | 12 ++++++--- .../parser/javacc/JdbcParseHandler.java | 27 +++++++++---------- .../internal/parser/javacc/ParseHandler.java | 2 +- .../src/main/javacc/ClickHouseSqlParser.jj | 7 ++--- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java index f623edda4..207c7ef89 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java @@ -66,8 +66,7 @@ public enum DriverProperties { *

  • JAVACC - parser extracts required information but PreparedStatement parameters parsed separately.
    • * */ -// SQL_PARSER("jdbc_sql_parser", "JAVACC", List.of("ANTLR4", "ANTLR4_PARAMS_PARSER", "JAVACC")), - SQL_PARSER("jdbc_sql_parser", "ANTLR4", List.of("ANTLR4", "ANTLR4_PARAMS_PARSER", "JAVACC")), + SQL_PARSER("jdbc_sql_parser", "JAVACC", List.of("ANTLR4", "ANTLR4_PARAMS_PARSER", "JAVACC")), ; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java index 27e5d83d8..284228595 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java @@ -107,7 +107,7 @@ public ParsedPreparedStatement parsePreparedStatement(String sql) { } } - stmt.setUseFunction(false); + stmt.setUseFunction(parsedStmt.isFuncUsed()); parseParameters(sql, stmt); return stmt; } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java index 7dfd22876..46a6c32e9 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java @@ -45,19 +45,20 @@ public class ClickHouseSqlStatement { private final Map settings; private final Set tempTables; private final int valueGroups; + private final boolean funcUsed; public ClickHouseSqlStatement(String sql) { - this(sql, StatementType.UNKNOWN, null, null, null, null, null, null, null, null, null, null, null, null, 0); + this(sql, StatementType.UNKNOWN, null, null, null, null, null, null, null, null, null, null, null, null,0, false); } public ClickHouseSqlStatement(String sql, StatementType stmtType) { - this(sql, stmtType, null, null, null, null, null, null, null, null, null, null, null, null, 0); + this(sql, stmtType, null, null, null, null, null, null, null, null, null, null, null, null, 0, false); } public ClickHouseSqlStatement(String sql, StatementType stmtType, String cluster, String database, String table, String input, String compressAlgorithm, String compressLevel, String format, String file, List parameters, Map positions, Map settings, - Set tempTables, int valueGroups) { + Set tempTables, int valueGroups, boolean funcUsed) { this.sql = sql; this.stmtType = stmtType; @@ -70,6 +71,7 @@ public ClickHouseSqlStatement(String sql, StatementType stmtType, String cluster this.format = format; this.file = file; this.valueGroups = valueGroups; + this.funcUsed = funcUsed; if (parameters != null && !parameters.isEmpty()) { this.parameters = Collections.unmodifiableList(parameters); @@ -144,6 +146,10 @@ public boolean isTCL() { return this.stmtType.getLanguageType() == LanguageType.TCL; } + public boolean isFuncUsed() { + return funcUsed; + } + public boolean isIdemponent() { boolean result = this.stmtType.isIdempotent() && !this.hasFile(); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java index 4f36c5729..05fb039ff 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java @@ -5,7 +5,6 @@ import com.clickhouse.data.ClickHouseFormat; import com.clickhouse.data.ClickHouseUtils; - import java.util.List; import java.util.Map; import java.util.Set; @@ -52,7 +51,7 @@ private void addMutationSetting(String sql, StringBuilder builder, Map parameters, Map positions, Map settings, - Set tempTables) { + Set tempTables, boolean funcUsed) { StringBuilder builder = new StringBuilder(); int index = positions.get("DELETE"); if (index > 0) { @@ -71,13 +70,13 @@ private ClickHouseSqlStatement handleDelete(String sql, StatementType stmtType, builder.append("TRUNCATE TABLE").append(sql.substring(index + 4)); } return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, - compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0); + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0, funcUsed); } private ClickHouseSqlStatement handleUpdate(String sql, StatementType stmtType, String cluster, String database, String table, String input, String compressAlgorithm, String compressLevel, String format, String file, List parameters, Map positions, Map settings, - Set tempTables) { + Set tempTables, boolean funcUsed) { StringBuilder builder = new StringBuilder(); int index = positions.get("UPDATE"); if (index > 0) { @@ -91,13 +90,13 @@ private ClickHouseSqlStatement handleUpdate(String sql, StatementType stmtType, builder.append('`').append(table).append('`').append(" UPDATE"); // .append(sql.substring(index + 3)); addMutationSetting(sql, builder, positions, settings, index + 3); return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, - compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0); + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0, funcUsed); } private ClickHouseSqlStatement handleInFileForInsertQuery(String sql, StatementType stmtType, String cluster, String database, String table, String input, String compressAlgorithm, String compressLevel, String format, String file, List parameters, Map positions, Map settings, - Set tempTables, int valueGroups) { + Set tempTables, int valueGroups, boolean funcUsed) { StringBuilder builder = new StringBuilder(sql.length()); builder.append(sql.substring(0, positions.get("FROM"))); Integer index = positions.get("SETTINGS"); @@ -115,13 +114,13 @@ private ClickHouseSqlStatement handleInFileForInsertQuery(String sql, StatementT builder.append("FORMAT ").append(format); } return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, - compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, valueGroups); + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, valueGroups, funcUsed); } private ClickHouseSqlStatement handleOutFileForSelectQuery(String sql, StatementType stmtType, String cluster, String database, String table, String input, String compressAlgorithm, String compressLevel, String format, String file, List parameters, Map positions, Map settings, - Set tempTables) { + Set tempTables, boolean funcUsed) { StringBuilder builder = new StringBuilder(sql.length()); builder.append(sql.substring(0, positions.get("INTO"))); Integer index = positions.get("FORMAT"); @@ -129,30 +128,30 @@ private ClickHouseSqlStatement handleOutFileForSelectQuery(String sql, Statement builder.append(sql.substring(index)); } return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input, - compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0); + compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0, funcUsed); } @Override public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType, String cluster, String database, String table, String input, String compressAlgorithm, String compressLevel, String format, String file, List parameters, Map positions, Map settings, - Set tempTables, int valueGroups) { + Set tempTables, int valueGroups, boolean funcUsed) { boolean hasFile = allowLocalFile && !ClickHouseChecker.isNullOrEmpty(file) && file.charAt(0) == '\''; ClickHouseSqlStatement s = null; if (stmtType == StatementType.DELETE) { s = allowLightWeightDelete ? s : handleDelete(sql, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, - format, file, parameters, positions, settings, tempTables); + format, file, parameters, positions, settings, tempTables, funcUsed); } else if (stmtType == StatementType.UPDATE) { s = allowLightWeightUpdate ? s : handleUpdate(sql, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, - format, file, parameters, positions, settings, tempTables); + format, file, parameters, positions, settings, tempTables, funcUsed); } else if (stmtType == StatementType.INSERT && hasFile) { s = handleInFileForInsertQuery(sql, stmtType, cluster, database, table, input, compressAlgorithm, - compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups); + compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups, funcUsed); } else if (stmtType == StatementType.SELECT && hasFile) { s = handleOutFileForSelectQuery(sql, stmtType, cluster, database, table, input, compressAlgorithm, - compressLevel, format, file, parameters, positions, settings, tempTables); + compressLevel, format, file, parameters, positions, settings, tempTables, funcUsed); } return s; } diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java index 6f1af61a6..5909fcc2d 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java @@ -52,7 +52,7 @@ public String handleParameter(String cluster, String database, String table, int public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType, String cluster, String database, String table, String input, String compressAlgorithm, String compressLevel, String format, String file, List parameters, Map positions, Map settings, - Set tempTables, int valueGroup) { + Set tempTables, int valueGroup, boolean funcUsed) { return null; } } diff --git a/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj b/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj index fae8e3cdd..cb9fac0ca 100644 --- a/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj +++ b/jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj @@ -175,6 +175,7 @@ TOKEN_MGR_DECLS: { String format = null; String file = null; int valueGroups = 0; + boolean funcUsed = false; final List parameters = new ArrayList<>(); final Map positions = new HashMap<>(); @@ -277,12 +278,12 @@ TOKEN_MGR_DECLS: { if (handler != null) { s = handler.handleStatement( - sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups); + sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups, funcUsed); } if (s == null) { s = new ClickHouseSqlStatement( - sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups); + sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups, funcUsed); } // reset variables @@ -811,7 +812,7 @@ void columnExpr(): { Token t; } { | LOOKAHEAD(3) literal() literal() | (LOOKAHEAD(2) macro())+ | LOOKAHEAD(2, { !(tokenIn(1, INF, NAN, NULL) && tokenIn(2, DOT)) }) literal() - | LOOKAHEAD(2, { getToken(2).kind == LPAREN }) functionExpr() + | LOOKAHEAD(2, { getToken(2).kind == LPAREN }) functionExpr() { token_source.funcUsed = true; } | anyIdentifier() (LOOKAHEAD(2) anyIdentifier())* }