diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index a5304e995bf030..2ee288d489487d 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -24,8 +24,8 @@ defaultDb = "regression_test" // init cmd like: select @@session.tx_read_only // at each time we connect. // add allowLoadLocalInfile so that the jdbc can execute mysql load data from client. -jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true" -targetJdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true" +jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" +targetJdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" jdbcUser = "root" jdbcPassword = "" diff --git a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy index fabf9f2e490797..4fe22d3d2cbfe4 100644 --- a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy +++ b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy @@ -17,6 +17,9 @@ testGroups = "p0" +jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" +targetJdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" + // exclude groups and exclude suites is more prior than include groups and include suites. // keep them in lexico order(add/remove cases between the sentinals and sort): // * sort lines in vim: select lines and then type :sort diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy index 0b8c3a2508934e..f89365ef4f4fac 100644 --- a/regression-test/pipeline/p0/conf/regression-conf.groovy +++ b/regression-test/pipeline/p0/conf/regression-conf.groovy @@ -20,8 +20,8 @@ // **Note**: default db will be create if not exist defaultDb = "regression_test" -jdbcUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLoadLocalInfile=true" -targetJdbcUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLoadLocalInfile=true" +jdbcUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" +targetJdbcUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" jdbcUser = "root" jdbcPassword = "" diff --git a/regression-test/suites/doc/table-design/data-partitioning/auto-partitioning.md.groovy b/regression-test/suites/doc/table-design/data-partitioning/auto-partitioning.md.groovy index b8b40c87cf033f..c67537d78c9a98 100644 --- a/regression-test/suites/doc/table-design/data-partitioning/auto-partitioning.md.groovy +++ b/regression-test/suites/doc/table-design/data-partitioning/auto-partitioning.md.groovy @@ -18,140 +18,162 @@ import org.junit.jupiter.api.Assertions; suite("docs/table-design/data-partitioning/auto-partitioning.md") { - try { - sql "drop table if exists `DAILY_TRADE_VALUE`" - sql """ - CREATE TABLE `DAILY_TRADE_VALUE` - ( - `TRADE_DATE` datev2 NOT NULL COMMENT '交易日期', - `TRADE_ID` varchar(40) NOT NULL COMMENT '交易编号', - ) - UNIQUE KEY(`TRADE_DATE`, `TRADE_ID`) - PARTITION BY RANGE(`TRADE_DATE`) - ( - PARTITION p_2000 VALUES [('2000-01-01'), ('2001-01-01')), - PARTITION p_2001 VALUES [('2001-01-01'), ('2002-01-01')), - PARTITION p_2002 VALUES [('2002-01-01'), ('2003-01-01')), - PARTITION p_2003 VALUES [('2003-01-01'), ('2004-01-01')), - PARTITION p_2004 VALUES [('2004-01-01'), ('2005-01-01')), - PARTITION p_2005 VALUES [('2005-01-01'), ('2006-01-01')), - PARTITION p_2006 VALUES [('2006-01-01'), ('2007-01-01')), - PARTITION p_2007 VALUES [('2007-01-01'), ('2008-01-01')), - PARTITION p_2008 VALUES [('2008-01-01'), ('2009-01-01')), - PARTITION p_2009 VALUES [('2009-01-01'), ('2010-01-01')), - PARTITION p_2010 VALUES [('2010-01-01'), ('2011-01-01')), - PARTITION p_2011 VALUES [('2011-01-01'), ('2012-01-01')), - PARTITION p_2012 VALUES [('2012-01-01'), ('2013-01-01')), - PARTITION p_2013 VALUES [('2013-01-01'), ('2014-01-01')), - PARTITION p_2014 VALUES [('2014-01-01'), ('2015-01-01')), - PARTITION p_2015 VALUES [('2015-01-01'), ('2016-01-01')), - PARTITION p_2016 VALUES [('2016-01-01'), ('2017-01-01')), - PARTITION p_2017 VALUES [('2017-01-01'), ('2018-01-01')), - PARTITION p_2018 VALUES [('2018-01-01'), ('2019-01-01')), - PARTITION p_2019 VALUES [('2019-01-01'), ('2020-01-01')), - PARTITION p_2020 VALUES [('2020-01-01'), ('2021-01-01')), - PARTITION p_2021 VALUES [('2021-01-01'), ('2022-01-01')) - ) - DISTRIBUTED BY HASH(`TRADE_DATE`) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ) - """ + sql "drop table if exists `DAILY_TRADE_VALUE`" + sql """ + CREATE TABLE `DAILY_TRADE_VALUE` + ( + `TRADE_DATE` datev2 NOT NULL COMMENT '交易日期', + `TRADE_ID` varchar(40) NOT NULL COMMENT '交易编号', + ) + UNIQUE KEY(`TRADE_DATE`, `TRADE_ID`) + PARTITION BY RANGE(`TRADE_DATE`) + ( + PARTITION p_2000 VALUES [('2000-01-01'), ('2001-01-01')), + PARTITION p_2001 VALUES [('2001-01-01'), ('2002-01-01')), + PARTITION p_2002 VALUES [('2002-01-01'), ('2003-01-01')), + PARTITION p_2003 VALUES [('2003-01-01'), ('2004-01-01')), + PARTITION p_2004 VALUES [('2004-01-01'), ('2005-01-01')), + PARTITION p_2005 VALUES [('2005-01-01'), ('2006-01-01')), + PARTITION p_2006 VALUES [('2006-01-01'), ('2007-01-01')), + PARTITION p_2007 VALUES [('2007-01-01'), ('2008-01-01')), + PARTITION p_2008 VALUES [('2008-01-01'), ('2009-01-01')), + PARTITION p_2009 VALUES [('2009-01-01'), ('2010-01-01')), + PARTITION p_2010 VALUES [('2010-01-01'), ('2011-01-01')), + PARTITION p_2011 VALUES [('2011-01-01'), ('2012-01-01')), + PARTITION p_2012 VALUES [('2012-01-01'), ('2013-01-01')), + PARTITION p_2013 VALUES [('2013-01-01'), ('2014-01-01')), + PARTITION p_2014 VALUES [('2014-01-01'), ('2015-01-01')), + PARTITION p_2015 VALUES [('2015-01-01'), ('2016-01-01')), + PARTITION p_2016 VALUES [('2016-01-01'), ('2017-01-01')), + PARTITION p_2017 VALUES [('2017-01-01'), ('2018-01-01')), + PARTITION p_2018 VALUES [('2018-01-01'), ('2019-01-01')), + PARTITION p_2019 VALUES [('2019-01-01'), ('2020-01-01')), + PARTITION p_2020 VALUES [('2020-01-01'), ('2021-01-01')), + PARTITION p_2021 VALUES [('2021-01-01'), ('2022-01-01')) + ) + DISTRIBUTED BY HASH(`TRADE_DATE`) BUCKETS 10 + PROPERTIES ( + "replication_num" = "1" + ) + """ - sql "drop table if exists `date_table`" - sql """ - CREATE TABLE `date_table` ( - `TIME_STAMP` datev2 NOT NULL COMMENT '采集日期' - ) ENGINE=OLAP - DUPLICATE KEY(`TIME_STAMP`) - AUTO PARTITION BY RANGE (date_trunc(`TIME_STAMP`, 'month')) - ( - ) - DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ) - """ + sql "drop table if exists `date_table`" + sql """ + CREATE TABLE `date_table` ( + `TIME_STAMP` datev2 NOT NULL COMMENT '采集日期' + ) ENGINE=OLAP + DUPLICATE KEY(`TIME_STAMP`) + AUTO PARTITION BY RANGE (date_trunc(`TIME_STAMP`, 'month')) + ( + ) + DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ) + """ + + sql "drop table if exists `str_table`" + sql """ + CREATE TABLE `str_table` ( + `str` varchar not null + ) ENGINE=OLAP + DUPLICATE KEY(`str`) + AUTO PARTITION BY LIST (`str`) + ( + ) + DISTRIBUTED BY HASH(`str`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ) + """ - sql "drop table if exists `str_table`" + sql "drop table if exists auto_null_list" + multi_sql """ + create table auto_null_list( + k0 varchar null + ) + auto partition by list (k0) + ( + ) + DISTRIBUTED BY HASH(`k0`) BUCKETS 1 + properties("replication_num" = "1"); + + insert into auto_null_list values (null); + select * from auto_null_list; + select * from auto_null_list partition(pX); + """ + + try { + sql "drop table if exists `range_table_nullable`" sql """ - CREATE TABLE `str_table` ( - `str` varchar not null + CREATE TABLE `range_table_nullable` ( + `k1` INT, + `k2` DATETIMEV2(3), + `k3` DATETIMEV2(6) ) ENGINE=OLAP - DUPLICATE KEY(`str`) - AUTO PARTITION BY LIST (`str`) + DUPLICATE KEY(`k1`) + AUTO PARTITION BY RANGE (date_trunc(`k2`, 'day')) ( ) - DISTRIBUTED BY HASH(`str`) BUCKETS 10 + DISTRIBUTED BY HASH(`k1`) BUCKETS 16 PROPERTIES ( "replication_allocation" = "tag.location.default: 1" ) """ + Assertions.fail("The SQL above should throw an exception as follows:\n\t\terrCode = 2, detailMessage = AUTO RANGE PARTITION doesn't support NULL column") + } catch (Exception e) { + assertTrue(e.getMessage().contains("errCode = 2, detailMessage = AUTO RANGE PARTITION doesn't support NULL column")) + } - sql "drop table if exists auto_null_list" - multi_sql """ - create table auto_null_list( - k0 varchar null - ) - auto partition by list (k0) - ( - ) - DISTRIBUTED BY HASH(`k0`) BUCKETS 1 - properties("replication_num" = "1"); + sql "create database if not exists auto_partition_doc_test" + sql "use auto_partition_doc_test" + sql "drop table if exists `DAILY_TRADE_VALUE`" + sql """ + CREATE TABLE `DAILY_TRADE_VALUE` + ( + `TRADE_DATE` datev2 NOT NULL COMMENT '交易日期', + `TRADE_ID` varchar(40) NOT NULL COMMENT '交易编号', + ) + UNIQUE KEY(`TRADE_DATE`, `TRADE_ID`) + AUTO PARTITION BY RANGE (date_trunc(`TRADE_DATE`, 'year')) + ( + ) + DISTRIBUTED BY HASH(`TRADE_DATE`) BUCKETS 10 + PROPERTIES ( + "replication_num" = "1" + ) + """ + def res1 = sql "show partitions from `DAILY_TRADE_VALUE`" + assertTrue(res1.isEmpty()) - insert into auto_null_list values (null); - select * from auto_null_list; - select * from auto_null_list partition(pX); - """ + def res2 = multi_sql """ + insert into `DAILY_TRADE_VALUE` values ('2012-12-13', 1), ('2008-02-03', 2), ('2014-11-11', 3); + show partitions from `DAILY_TRADE_VALUE`; + """ + assertTrue(res2[1].size() == 3) - try { - sql "drop table if exists `range_table_nullable`" - sql """ - CREATE TABLE `range_table_nullable` ( - `k1` INT, - `k2` DATETIMEV2(3), - `k3` DATETIMEV2(6) - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - AUTO PARTITION BY RANGE (date_trunc(`k2`, 'day')) - ( - ) - DISTRIBUTED BY HASH(`k1`) BUCKETS 16 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ) - """ - Assertions.fail("The SQL above should throw an exception as follows:\n\t\terrCode = 2, detailMessage = AUTO RANGE PARTITION doesn't support NULL column") - } catch (Exception e) { - assertTrue(e.getMessage().contains("errCode = 2, detailMessage = AUTO RANGE PARTITION doesn't support NULL column")) - } + sql """ + select * from partitions("catalog"="internal","database"="auto_partition_doc_test","table"="DAILY_TRADE_VALUE") where PartitionName = auto_partition_name('range', 'year', '2008-02-03'); + """ - sql "drop table if exists `DAILY_TRADE_VALUE`" - sql """ - CREATE TABLE `DAILY_TRADE_VALUE` - ( - `TRADE_DATE` datev2 NOT NULL COMMENT '交易日期', - `TRADE_ID` varchar(40) NOT NULL COMMENT '交易编号', - ) - UNIQUE KEY(`TRADE_DATE`, `TRADE_ID`) - AUTO PARTITION BY RANGE (date_trunc(`TRADE_DATE`, 'year')) - ( - ) - DISTRIBUTED BY HASH(`TRADE_DATE`) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ) - """ - def res1 = sql "show partitions from `DAILY_TRADE_VALUE`" - assertTrue(res1.isEmpty()) - def res2 = multi_sql """ - insert into `DAILY_TRADE_VALUE` values ('2012-12-13', 1), ('2008-02-03', 2), ('2014-11-11', 3); - show partitions from `DAILY_TRADE_VALUE`; - """ - assertTrue(res2[1].size() == 3) - } catch (Throwable t) { - Assertions.fail("examples in docs/table-design/data-partitioning/auto-partitioning.md failed to exec, please fix it", t) - } + sql "drop table if exists auto_dynamic" + sql """ + create table auto_dynamic( + k0 datetime(6) NOT NULL + ) + auto partition by range (date_trunc(k0, 'year')) + ( + ) + DISTRIBUTED BY HASH(`k0`) BUCKETS 2 + properties( + "dynamic_partition.enable" = "true", + "dynamic_partition.prefix" = "p", + "dynamic_partition.start" = "-50", + "dynamic_partition.end" = "0", --- Dynamic Partition 不创建分区 + "dynamic_partition.time_unit" = "year", + "replication_num" = "1" + ); + """ } diff --git a/regression-test/suites/doc/table-design/data-partitioning/basic-concepts.md.groovy b/regression-test/suites/doc/table-design/data-partitioning/basic-concepts.md.groovy index b257b434a05c70..8558aadb218cbc 100644 --- a/regression-test/suites/doc/table-design/data-partitioning/basic-concepts.md.groovy +++ b/regression-test/suites/doc/table-design/data-partitioning/basic-concepts.md.groovy @@ -18,43 +18,208 @@ import org.junit.jupiter.api.Assertions; suite("docs/table-design/data-partitioning/basic-concepts.md") { - try { - sql "drop table if exists example_range_tbl" - multi_sql """ - -- Range Partition - CREATE TABLE IF NOT EXISTS example_range_tbl - ( - `user_id` LARGEINT NOT NULL COMMENT "用户id", - `date` DATE NOT NULL COMMENT "数据灌入日期时间", - `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", - `city` VARCHAR(20) COMMENT "用户所在城市", - `age` SMALLINT COMMENT "用户年龄", - `sex` TINYINT COMMENT "用户性别", - `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", - `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", - `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", - `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" - ) - ENGINE=OLAP - AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) - PARTITION BY RANGE(`date`) - ( - PARTITION `p201701` VALUES LESS THAN ("2017-02-01"), - PARTITION `p201702` VALUES LESS THAN ("2017-03-01"), - PARTITION `p201703` VALUES LESS THAN ("2017-04-01"), - PARTITION `p2018` VALUES [("2018-01-01"), ("2019-01-01")) - ) - DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 - PROPERTIES - ( - "replication_num" = "1" - ); - """ - - sql "show create table example_range_tbl" - sql "show partitions from example_range_tbl" - sql """ALTER TABLE example_range_tbl ADD PARTITION p201704 VALUES LESS THAN("2020-05-01") DISTRIBUTED BY HASH(`user_id`) BUCKETS 5""" - } catch (Throwable t) { - Assertions.fail("examples in docs/table-design/data-partitioning/basic-concepts.md failed to exec, please fix it", t) - } + sql "drop table if exists example_range_tbl" + sql """ + -- Range Partition + CREATE TABLE IF NOT EXISTS example_range_tbl + ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" + ) + ENGINE=OLAP + AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) + PARTITION BY RANGE(`date`) + ( + PARTITION `p201701` VALUES LESS THAN ("2017-02-01"), + PARTITION `p201702` VALUES LESS THAN ("2017-03-01"), + PARTITION `p201703` VALUES LESS THAN ("2017-04-01"), + PARTITION `p2018` VALUES [("2018-01-01"), ("2019-01-01")) + ) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 + PROPERTIES + ( + "replication_num" = "1" + ); + """ + + sql "show create table example_range_tbl" + sql "show partitions from example_range_tbl" + sql """ALTER TABLE example_range_tbl ADD PARTITION p201704 VALUES LESS THAN("2020-05-01") DISTRIBUTED BY HASH(`user_id`) BUCKETS 5""" + + + + // table items + sql "drop table if exists example_range_tbl" + sql """ + CREATE TABLE IF NOT EXISTS example_range_tbl + ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" + ) + ENGINE=OLAP + AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) + AUTO PARTITION BY RANGE(date_trunc(`date`, 'month')) --- 使用月作为分区粒度 + () + DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 + PROPERTIES + ( + "replication_num" = "1" + ); + """ + + sql "drop table if exists example_range_tbl" + sql """ + CREATE TABLE IF NOT EXISTS example_range_tbl + ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" + ) + ENGINE=OLAP + AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) + PARTITION BY RANGE(`date`) + () + DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 + PROPERTIES + ( + "replication_num" = "1", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "WEEK", --- 分区粒度为周 + "dynamic_partition.start" = "-2", --- 向前保留两周 + "dynamic_partition.end" = "2", --- 提前创建后两周 + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "8" + ); + """ + + sql "drop table if exists example_range_tbl" + sql """ + CREATE TABLE IF NOT EXISTS example_range_tbl + ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" + ) + ENGINE=OLAP + AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) + AUTO PARTITION BY RANGE(date_trunc(`date`, 'month')) --- 使用月作为分区粒度 + () + DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 + PROPERTIES + ( + "replication_num" = "1", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "month", --- 二者粒度必须相同 + "dynamic_partition.start" = "-2", --- 动态分区自动清理超过两周的历史分区 + "dynamic_partition.end" = "0", --- 动态分区不创建未来分区,完全交给自动分区 + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "8" + ); + """ + + sql "drop table if exists example_range_tbl" + sql """ + CREATE TABLE IF NOT EXISTS example_range_tbl + ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间" + ) + ENGINE=OLAP + AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) + PARTITION BY RANGE(`date`) + ( + PARTITION `p201701` VALUES LESS THAN ("2017-02-01"), + PARTITION `p201702` VALUES LESS THAN ("2017-03-01"), + PARTITION `p201703` VALUES LESS THAN ("2017-04-01"), + PARTITION `p2018` VALUES [("2018-01-01"), ("2019-01-01")) + ) + DISTRIBUTED BY HASH(`user_id`) BUCKETS AUTO + PROPERTIES + ( + "replication_num" = "1", + "estimate_partition_size" = "2G" --- 用户估计一个分区将有的数据量,不提供则默认为 10G + ); + """ + + + + // Partition Retrieval + sql "create database if not exists partition_basic_concepts" + sql "use partition_basic_concepts" + sql "drop table if exists DAILY_TRADE_VALUE" + sql """ + CREATE TABLE `DAILY_TRADE_VALUE` + ( + `TRADE_DATE` datev2 NOT NULL COMMENT '交易日期', + `TRADE_ID` varchar(40) NOT NULL COMMENT '交易编号', + ) + UNIQUE KEY(`TRADE_DATE`, `TRADE_ID`) + AUTO PARTITION BY RANGE (date_trunc(`TRADE_DATE`, 'year')) + ( + ) + DISTRIBUTED BY HASH(`TRADE_DATE`) BUCKETS 10 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql "drop table if exists list_table1" + sql """ + CREATE TABLE `list_table1` ( + `str` varchar + ) ENGINE=OLAP + DUPLICATE KEY(`str`) + COMMENT 'OLAP' + AUTO PARTITION BY LIST (`str`) + ( + ) + DISTRIBUTED BY HASH(`str`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """ insert into list_table1 values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc"), (null) """ + sql """ insert into list_table1 values (null), ("") """ // not same partition + + sql """ select * from partitions("catalog"="internal", "database"="partition_basic_concepts", "table"="DAILY_TRADE_VALUE") where PartitionName = auto_partition_name('range', 'year', '2008-02-03'); """ + sql """ select * from information_schema.partitions where TABLE_SCHEMA='partition_basic_concepts' and TABLE_NAME='list_table1' and PARTITION_NAME=auto_partition_name('list', null); """ + sql """ select * from information_schema.partitions where TABLE_NAME='DAILY_TRADE_VALUE' and PARTITION_DESCRIPTION like "[('2012-01-01'),%"; """ }