From 5aeccb7a095cf6ff4d51d4b3263aee2a9005ee16 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 22 Mar 2024 17:12:41 +0800 Subject: [PATCH 01/12] 1 --- .../org/apache/doris/catalog/OlapTable.java | 2 +- .../mtmv_p0/test_null_partition_mtmv.groovy | 66 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index b6239f486cdeec..471814ecf2e63a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2726,6 +2726,6 @@ public boolean needAutoRefresh() { @Override public boolean isPartitionColumnAllowNull() { - return false; + return true; } } diff --git a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy new file mode 100644 index 00000000000000..dfa6b438f8ec9a --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_null_partition_mtmv") { + def tableName = "t_test_null_partition_mtmv_user" + def mvName = "t_test_null_partition_mtmv" + def dbName = "regression_test_mtmv_p0" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + + + sql """ + CREATE TABLE `${tableName}` ( + `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"', + `num` SMALLINT COMMENT '\"数量\"' + ) ENGINE=OLAP + DUPLICATE KEY(`user_id`, `num`) + COMMENT 'OLAP' + PARTITION BY list(`num`) + ( + PARTITION p_1 VALUES IN (1), + PARTITION p_null VALUES IN (null) + ) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + insert into ${tableNameNum} values(1,1),(2,null); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`num`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * FROM ${tableNameNum}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertTrue(showPartitionsResult.toString().contains("p_1")) + assertTrue(showPartitionsResult.toString().contains("p_null")) + + order_qt_list_null "SELECT * FROM ${mvName} partitions(p_null) order by user_id,num" + order_qt_list_1 "SELECT * FROM ${mvName} partitions(p_1) order by user_id,num" + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} From 60a8009375e0e480dd55b97e00e97bd63054b8dc Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 22 Mar 2024 17:27:27 +0800 Subject: [PATCH 02/12] 1 --- .../suites/mtmv_p0/test_null_partition_mtmv.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy index dfa6b438f8ec9a..07baacc756ce4a 100644 --- a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy @@ -41,7 +41,7 @@ suite("test_null_partition_mtmv") { PROPERTIES ('replication_num' = '1') ; """ sql """ - insert into ${tableNameNum} values(1,1),(2,null); + insert into ${tableName} values(1,1),(2,null); """ sql """ @@ -51,7 +51,7 @@ suite("test_null_partition_mtmv") { DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS - SELECT * FROM ${tableNameNum}; + SELECT * FROM ${tableName}; """ showPartitionsResult = sql """show partitions from ${mvName}""" logger.info("showPartitionsResult: " + showPartitionsResult.toString()) From 69edb98f2772c73e8488b0591bf39abde9d84503 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 22 Mar 2024 17:31:43 +0800 Subject: [PATCH 03/12] 1 --- regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy index 07baacc756ce4a..06a8029c6e17f1 100644 --- a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy @@ -56,7 +56,7 @@ suite("test_null_partition_mtmv") { showPartitionsResult = sql """show partitions from ${mvName}""" logger.info("showPartitionsResult: " + showPartitionsResult.toString()) assertTrue(showPartitionsResult.toString().contains("p_1")) - assertTrue(showPartitionsResult.toString().contains("p_null")) + assertTrue(showPartitionsResult.toString().contains("p_NULL")) order_qt_list_null "SELECT * FROM ${mvName} partitions(p_null) order by user_id,num" order_qt_list_1 "SELECT * FROM ${mvName} partitions(p_1) order by user_id,num" From 5f03eae6869bfadb2d0ec07596ae36671a447b95 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 22 Mar 2024 17:31:54 +0800 Subject: [PATCH 04/12] 1 --- regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy index 06a8029c6e17f1..69cb4ca46802e7 100644 --- a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy @@ -58,7 +58,7 @@ suite("test_null_partition_mtmv") { assertTrue(showPartitionsResult.toString().contains("p_1")) assertTrue(showPartitionsResult.toString().contains("p_NULL")) - order_qt_list_null "SELECT * FROM ${mvName} partitions(p_null) order by user_id,num" + order_qt_list_null "SELECT * FROM ${mvName} partitions(p_NULL) order by user_id,num" order_qt_list_1 "SELECT * FROM ${mvName} partitions(p_1) order by user_id,num" sql """drop table if exists `${tableName}`""" From 216b0bf4743563af863aa227b8500843d03a1ed9 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 22 Mar 2024 17:34:14 +0800 Subject: [PATCH 05/12] 1 --- .../suites/mtmv_p0/test_null_partition_mtmv.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy index 69cb4ca46802e7..df9ad5e4a99343 100644 --- a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy @@ -58,6 +58,13 @@ suite("test_null_partition_mtmv") { assertTrue(showPartitionsResult.toString().contains("p_1")) assertTrue(showPartitionsResult.toString().contains("p_NULL")) + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + def jobName = getJobName(dbName, mvName); + log.info(jobName) + waitingMTMVTaskFinished(jobName) + order_qt_list_null "SELECT * FROM ${mvName} partitions(p_NULL) order by user_id,num" order_qt_list_1 "SELECT * FROM ${mvName} partitions(p_1) order by user_id,num" From 50620c4d2089bf4b74f6824c08ebb50a1f635e3b Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 22 Mar 2024 17:43:54 +0800 Subject: [PATCH 06/12] 1 --- .../mtmv_p0/test_null_partition_mtmv.groovy | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy index df9ad5e4a99343..1e5df7547fb226 100644 --- a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy @@ -24,7 +24,7 @@ suite("test_null_partition_mtmv") { sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" - + // list table sql """ CREATE TABLE `${tableName}` ( `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"', @@ -53,7 +53,7 @@ suite("test_null_partition_mtmv") { AS SELECT * FROM ${tableName}; """ - showPartitionsResult = sql """show partitions from ${mvName}""" + def showPartitionsResult = sql """show partitions from ${mvName}""" logger.info("showPartitionsResult: " + showPartitionsResult.toString()) assertTrue(showPartitionsResult.toString().contains("p_1")) assertTrue(showPartitionsResult.toString().contains("p_NULL")) @@ -70,4 +70,53 @@ suite("test_null_partition_mtmv") { sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" + + + // range table + sql """ + CREATE TABLE `${tableName}` ( + `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"', + `num` SMALLINT COMMENT '\"数量\"' + ) ENGINE=OLAP + DUPLICATE KEY(`user_id`, `num`) + COMMENT 'OLAP' + PARTITION BY range(`num`) + ( + PARTITION p_10 VALUES LESS THAN (10), + PARTITION p_20 VALUES LESS THAN (20) + ) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + insert into ${tableName} values(1,null),(2,15); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`num`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * FROM ${tableName}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertTrue(showPartitionsResult.toString().contains("p_32768_10")) + assertTrue(showPartitionsResult.toString().contains("p_10_20")) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + jobName = getJobName(dbName, mvName); + log.info(jobName) + waitingMTMVTaskFinished(jobName) + + order_qt_range_p_32768_10 "SELECT * FROM ${mvName} partitions(p_32768_10) order by user_id,num" + order_qt_list_p_10_20 "SELECT * FROM ${mvName} partitions(p_10_20) order by user_id,num" + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + } From ebae2d6b6df1a409a5e9671f5fa21afad42ba7a8 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Tue, 26 Mar 2024 16:52:29 +0800 Subject: [PATCH 07/12] 1 --- .../data/mtmv_p0/test_null_partition_mtmv.out | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 regression-test/data/mtmv_p0/test_null_partition_mtmv.out diff --git a/regression-test/data/mtmv_p0/test_null_partition_mtmv.out b/regression-test/data/mtmv_p0/test_null_partition_mtmv.out new file mode 100644 index 00000000000000..0510a5b9d08752 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_null_partition_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !list_null -- +2 \N + +-- !list_1 -- +1 1 + +-- !range_p_32768_10 -- +1 \N + +-- !list_p_10_20 -- +2 15 + From dc00996a3d396c3492cd63fa5fc1b54e6e1805ca Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Tue, 26 Mar 2024 18:01:03 +0800 Subject: [PATCH 08/12] 1 --- .../nereids/rules/exploration/mv/MaterializedViewUtilsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 02fb18edbf7d15..8416bd396b04fc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -269,7 +269,7 @@ public void getRelatedTableInfoTestWithoutGroupNullTest() { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); Optional relatedTableInfo = MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan); - Assertions.assertFalse(relatedTableInfo.isPresent()); + Assertions.assertTrue(relatedTableInfo.isPresent()); }); } From 979266929cbf181b787d1b247f1126d4d10ac75b Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Wed, 27 Mar 2024 15:58:22 +0800 Subject: [PATCH 09/12] 1 --- .../mtmv_p0/test_null_partition_mtmv.groovy | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy index 1e5df7547fb226..5783bee731fe4c 100644 --- a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy @@ -119,4 +119,51 @@ suite("test_null_partition_mtmv") { sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" + // range table + sql """ + CREATE TABLE `${tableName}` ( + `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"', + `create_date` DATE + ) ENGINE=OLAP + DUPLICATE KEY(`user_id`, `create_date`) + COMMENT 'OLAP' + PARTITION BY range(`create_date`) + ( + PARTITION p_10 VALUES LESS THAN ("2020-11-11"), + PARTITION p_20 VALUES LESS THAN ("2021-11-11") + ) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + insert into ${tableName} values(1,null),(2,"2021-01-01"); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`create_date`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * FROM ${tableName}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertTrue(showPartitionsResult.toString().contains("p_32768_10")) + assertTrue(showPartitionsResult.toString().contains("p_10_20")) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + jobName = getJobName(dbName, mvName); + log.info(jobName) + waitingMTMVTaskFinished(jobName) + + order_qt_range_p_32768_10 "SELECT * FROM ${mvName} partitions(p_32768_10) order by user_id,num" + order_qt_list_p_10_20 "SELECT * FROM ${mvName} partitions(p_10_20) order by user_id,num" + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + } From 96c4e7e139834291480d65072ed240275c936b01 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Wed, 27 Mar 2024 16:02:28 +0800 Subject: [PATCH 10/12] 1 --- .../suites/mtmv_p0/test_null_partition_mtmv.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy index 5783bee731fe4c..2e34d2f8ff7bb4 100644 --- a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy @@ -150,8 +150,8 @@ suite("test_null_partition_mtmv") { """ showPartitionsResult = sql """show partitions from ${mvName}""" logger.info("showPartitionsResult: " + showPartitionsResult.toString()) - assertTrue(showPartitionsResult.toString().contains("p_32768_10")) - assertTrue(showPartitionsResult.toString().contains("p_10_20")) + assertTrue(showPartitionsResult.toString().contains("p_00000101_20201111")) + assertTrue(showPartitionsResult.toString().contains("p_20201111_20211111")) sql """ REFRESH MATERIALIZED VIEW ${mvName} AUTO @@ -160,8 +160,8 @@ suite("test_null_partition_mtmv") { log.info(jobName) waitingMTMVTaskFinished(jobName) - order_qt_range_p_32768_10 "SELECT * FROM ${mvName} partitions(p_32768_10) order by user_id,num" - order_qt_list_p_10_20 "SELECT * FROM ${mvName} partitions(p_10_20) order by user_id,num" + order_qt_range_p_00000101_20201111 "SELECT * FROM ${mvName} partitions(p_00000101_20201111) order by user_id,num" + order_qt_list_p_20201111_20211111 "SELECT * FROM ${mvName} partitions(p_20201111_20211111) order by user_id,num" sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" From f179a60e2680e03042947e9cee7097635345447f Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Wed, 27 Mar 2024 16:05:29 +0800 Subject: [PATCH 11/12] 1 --- .../suites/mtmv_p0/test_null_partition_mtmv.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy index 2e34d2f8ff7bb4..a6a7890180ecc2 100644 --- a/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_null_partition_mtmv.groovy @@ -114,7 +114,7 @@ suite("test_null_partition_mtmv") { waitingMTMVTaskFinished(jobName) order_qt_range_p_32768_10 "SELECT * FROM ${mvName} partitions(p_32768_10) order by user_id,num" - order_qt_list_p_10_20 "SELECT * FROM ${mvName} partitions(p_10_20) order by user_id,num" + order_qt_range_p_10_20 "SELECT * FROM ${mvName} partitions(p_10_20) order by user_id,num" sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" @@ -160,8 +160,8 @@ suite("test_null_partition_mtmv") { log.info(jobName) waitingMTMVTaskFinished(jobName) - order_qt_range_p_00000101_20201111 "SELECT * FROM ${mvName} partitions(p_00000101_20201111) order by user_id,num" - order_qt_list_p_20201111_20211111 "SELECT * FROM ${mvName} partitions(p_20201111_20211111) order by user_id,num" + order_qt_range_p_00000101_20201111 "SELECT * FROM ${mvName} partitions(p_00000101_20201111) order by user_id,create_date" + order_qt_range_p_20201111_20211111 "SELECT * FROM ${mvName} partitions(p_20201111_20211111) order by user_id,create_date" sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" From 7babbb32d7e1f59ea294b11e639067b312c9389f Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Wed, 27 Mar 2024 16:07:34 +0800 Subject: [PATCH 12/12] 1 --- regression-test/data/mtmv_p0/test_null_partition_mtmv.out | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/regression-test/data/mtmv_p0/test_null_partition_mtmv.out b/regression-test/data/mtmv_p0/test_null_partition_mtmv.out index 0510a5b9d08752..f341905773fc60 100644 --- a/regression-test/data/mtmv_p0/test_null_partition_mtmv.out +++ b/regression-test/data/mtmv_p0/test_null_partition_mtmv.out @@ -8,6 +8,12 @@ -- !range_p_32768_10 -- 1 \N --- !list_p_10_20 -- +-- !range_p_10_20 -- 2 15 +-- !range_p_00000101_20201111 -- +1 \N + +-- !range_p_20201111_20211111 -- +2 2021-01-01 +