From 1b191f032ff14147bc7b9c8bccbbf08936567a33 Mon Sep 17 00:00:00 2001 From: Rongqian Li Date: Sun, 15 May 2022 17:59:37 +0800 Subject: [PATCH 1/5] ADD: group_concat support distinct --- .../aggregate-functions/group_concat.md | 14 +++++++++++--- .../aggregate-functions/group_concat.md | 12 ++++++++++-- .../apache/doris/analysis/FunctionCallExpr.java | 4 ---- .../org/apache/doris/common/util/SqlUtils.java | 2 +- .../org/apache/doris/analysis/SelectStmtTest.java | 4 +++- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-manual/sql-functions/aggregate-functions/group_concat.md b/docs/en/sql-manual/sql-functions/aggregate-functions/group_concat.md index 7f7ec160e9ae58..7e0095cafda1ef 100644 --- a/docs/en/sql-manual/sql-functions/aggregate-functions/group_concat.md +++ b/docs/en/sql-manual/sql-functions/aggregate-functions/group_concat.md @@ -28,7 +28,7 @@ under the License. ### description #### Syntax -`VARCHAR GROUP_CONCAT(VARCHAR str[, VARCHAR sep])` +`VARCHAR GROUP_CONCAT([DISTINCT] VARCHAR str[, VARCHAR sep])` This function is an aggregation function similar to sum (), and group_concat links multiple rows of results in the result set to a string. The second parameter, sep, is a connection symbol between strings, which can be omitted. This function usually needs to be used with group by statements. @@ -43,22 +43,30 @@ mysql> select value from test; | a | | b | | c | +| c | +-------+ mysql> select GROUP_CONCAT(value) from test; +-----------------------+ | GROUP_CONCAT(`value`) | +-----------------------+ -| a, b, c | +| a, b, c, c | +-----------------------+ mysql> select GROUP_CONCAT(value, " ") from test; +----------------------------+ | GROUP_CONCAT(`value`, ' ') | +----------------------------+ -| a b c | +| a b c c | +----------------------------+ +mysql> select GROUP_CONCAT(DISTINCT value) from test; ++-----------------------+ +| GROUP_CONCAT(`value`) | ++-----------------------+ +| a, b, c | ++-----------------------+ + mysql> select GROUP_CONCAT(value, NULL) from test; +----------------------------+ | GROUP_CONCAT(`value`, NULL)| diff --git a/docs/zh-CN/sql-manual/sql-functions/aggregate-functions/group_concat.md b/docs/zh-CN/sql-manual/sql-functions/aggregate-functions/group_concat.md index 3a92c31f233fe8..ef6189483f34dd 100644 --- a/docs/zh-CN/sql-manual/sql-functions/aggregate-functions/group_concat.md +++ b/docs/zh-CN/sql-manual/sql-functions/aggregate-functions/group_concat.md @@ -28,7 +28,7 @@ under the License. ### description #### Syntax -`VARCHAR GROUP_CONCAT(VARCHAR str[, VARCHAR sep])` +`VARCHAR GROUP_CONCAT([DISTINCT] VARCHAR str[, VARCHAR sep])` 该函数是类似于 sum() 的聚合函数，group_concat 将结果集中的多行结果连接成一个字符串。第二个参数 sep 为字符串之间的连接符号，该参数可以省略。该函数通常需要和 group by 语句一起使用。 @@ -43,12 +43,20 @@ mysql> select value from test; | a | | b | | c | +| c | +-------+ mysql> select GROUP_CONCAT(value) from test; +-----------------------+ | GROUP_CONCAT(`value`) | +-----------------------+ +| a, b, c, c | ++-----------------------+ + +mysql> select GROUP_CONCAT(DISTINCT value) from test; ++-----------------------+ +| GROUP_CONCAT(`value`) | ++-----------------------+ | a, b, c | +-----------------------+ @@ -56,7 +64,7 @@ mysql> select GROUP_CONCAT(value, " ") from test; +----------------------------+ | GROUP_CONCAT(`value`, ' ') | +----------------------------+ -| a b c | +| a b c c | +----------------------------+ mysql> select GROUP_CONCAT(value, NULL) from test; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 40387673bdf013..7f55d59fe4dbec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -481,10 +481,6 @@ private void analyzeBuiltinAggFunction(Analyzer analyzer) throws AnalysisExcepti "group_concat requires one or two parameters: " + this.toSql()); } - if (fnParams.isDistinct()) { - throw new AnalysisException("group_concat does not support DISTINCT"); - } - Expr arg0 = getChild(0); if (!arg0.type.isStringType() && !arg0.type.isNull()) { throw new AnalysisException( diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/SqlUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/SqlUtils.java index 1fc04fc108a4b5..669088320920cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/SqlUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/SqlUtils.java @@ -17,7 +17,7 @@ package org.apache.doris.common.util; -import org.apache.parquet.Strings; +import com.google.common.base.Strings; public class SqlUtils { public static String escapeUnquote(String ident) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java index 4699a85792ee3d..7daacb70d5eb10 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java @@ -787,9 +787,11 @@ public void testWithUnionToSql() throws Exception { @Test public void testSelectOuterJoinSql() throws Exception { ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String sql1 = "select l.citycode, group_concat(r.username) from db1.table1 l left join db1.table2 r on l.citycode=r.citycode group by l.citycode"; + String sql1 = "select l.citycode, group_concat(distinct r.username) from db1.table1 l left join db1.table2 r on l.citycode=r.citycode group by l.citycode"; SelectStmt stmt1 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql1, ctx); Assert.assertTrue(stmt1.getAnalyzer().getSlotDesc(new SlotId(2)).getIsNullable()); Assert.assertTrue(stmt1.getAnalyzer().getSlotDescriptor("r.username").getIsNullable()); + FunctionCallExpr expr = (FunctionCallExpr) stmt1.getSelectList().getItems().get(1).getExpr(); + Assert.assertTrue(expr.getFnParams().isDistinct()); } } From 080c3684ea64e718e119e2c4ff40b7230087532c Mon Sep 17 00:00:00 2001 From: Rongqian Li Date: Mon, 16 May 2022 09:52:34 +0800 Subject: [PATCH 2/5] MOD: checkstyle --- .../test/java/org/apache/doris/analysis/SelectStmtTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java index 7daacb70d5eb10..0c82aed559eac7 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java @@ -787,7 +787,8 @@ public void testWithUnionToSql() throws Exception { @Test public void testSelectOuterJoinSql() throws Exception { ConnectContext ctx = UtFrameUtils.createDefaultCtx(); - String sql1 = "select l.citycode, group_concat(distinct r.username) from db1.table1 l left join db1.table2 r on l.citycode=r.citycode group by l.citycode"; + String sql1 = "select l.citycode, group_concat(distinct r.username) from db1.table1 l " + + "left join db1.table2 r on l.citycode=r.citycode group by l.citycode"; SelectStmt stmt1 = (SelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql1, ctx); Assert.assertTrue(stmt1.getAnalyzer().getSlotDesc(new SlotId(2)).getIsNullable()); Assert.assertTrue(stmt1.getAnalyzer().getSlotDescriptor("r.username").getIsNullable()); From 75bb39b076f1a662e5a585334aa78a067e4c7a8e Mon Sep 17 00:00:00 2001 From: stalary Date: Tue, 17 May 2022 10:45:18 +0800 Subject: [PATCH 3/5] ADD: add regression test --- .../group_concat/test_grouping_concat.out | 7 +++++ .../group_concat/test_grouping_concat.groovy | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 regression-test/data/query/group_concat/test_grouping_concat.out create mode 100644 regression-test/suites/query/group_concat/test_grouping_concat.groovy diff --git a/regression-test/data/query/group_concat/test_grouping_concat.out b/regression-test/data/query/group_concat/test_grouping_concat.out new file mode 100644 index 00000000000000..794453c5922a2d --- /dev/null +++ b/regression-test/data/query/group_concat/test_grouping_concat.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +true, false, false + +-- !select -- +false, true + diff --git a/regression-test/suites/query/group_concat/test_grouping_concat.groovy b/regression-test/suites/query/group_concat/test_grouping_concat.groovy new file mode 100644 index 00000000000000..a316255c823549 --- /dev/null +++ b/regression-test/suites/query/group_concat/test_grouping_concat.groovy @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_grouping_concat", "query") { + qt_select """ + SELECT group_concat(k6) FROM test_query_db.test + """ + + qt_select """ + SELECT group_concat(DISTINCT k6) FROM test_query_db.test + """ +} From dce82ae22fb37db3b74f10e24d12ef4508029cd3 Mon Sep 17 00:00:00 2001 From: stalary Date: Tue, 17 May 2022 14:33:08 +0800 Subject: [PATCH 4/5] MOD: delete regression test --- .../group_concat/test_grouping_concat.out | 7 ----- .../group_concat/test_grouping_concat.groovy | 26 ------------------- 2 files changed, 33 deletions(-) delete mode 100644 regression-test/data/query/group_concat/test_grouping_concat.out delete mode 100644 regression-test/suites/query/group_concat/test_grouping_concat.groovy diff --git a/regression-test/data/query/group_concat/test_grouping_concat.out b/regression-test/data/query/group_concat/test_grouping_concat.out deleted file mode 100644 index 794453c5922a2d..00000000000000 --- a/regression-test/data/query/group_concat/test_grouping_concat.out +++ /dev/null @@ -1,7 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select -- -true, false, false - --- !select -- -false, true - diff --git a/regression-test/suites/query/group_concat/test_grouping_concat.groovy b/regression-test/suites/query/group_concat/test_grouping_concat.groovy deleted file mode 100644 index a316255c823549..00000000000000 --- a/regression-test/suites/query/group_concat/test_grouping_concat.groovy +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_grouping_concat", "query") { - qt_select """ - SELECT group_concat(k6) FROM test_query_db.test - """ - - qt_select """ - SELECT group_concat(DISTINCT k6) FROM test_query_db.test - """ -} From c6643832663ab709180f4d9b435a03083a019be4 Mon Sep 17 00:00:00 2001 From: stalary Date: Tue, 17 May 2022 14:48:43 +0800 Subject: [PATCH 5/5] ADD: regression test --- .../query/group_concat/test_group_concat.out | 7 +++++ .../group_concat/test_group_concat.groovy | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 regression-test/data/query/group_concat/test_group_concat.out create mode 100644 regression-test/suites/query/group_concat/test_group_concat.groovy diff --git a/regression-test/data/query/group_concat/test_group_concat.out b/regression-test/data/query/group_concat/test_group_concat.out new file mode 100644 index 00000000000000..94f73cc5368d50 --- /dev/null +++ b/regression-test/data/query/group_concat/test_group_concat.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +false, false + +-- !select -- +false + diff --git a/regression-test/suites/query/group_concat/test_group_concat.groovy b/regression-test/suites/query/group_concat/test_group_concat.groovy new file mode 100644 index 00000000000000..6bb57dea447bf8 --- /dev/null +++ b/regression-test/suites/query/group_concat/test_group_concat.groovy @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_group_concat", "query") { + qt_select """ + SELECT group_concat(k6) FROM test_query_db.test where k6='false' + """ + + qt_select """ + SELECT group_concat(DISTINCT k6) FROM test_query_db.test where k6='false' + """ +}