diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 934aab251a7038..dcc3fdc0197e33 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -28,6 +28,8 @@ feHttpAddress = "127.0.0.1:8030" feHttpUser = "root" feHttpPassword = "" +beHttpAddress = "127.0.0.1:8040" + // set DORIS_HOME by system properties // e.g. java -DDORIS_HOME=./ suitePath = "${DORIS_HOME}/regression-test/suites" diff --git a/regression-test/data/compaction/test_compaction.out b/regression-test/data/compaction/test_compaction.out new file mode 100644 index 00000000000000..41365e84bfa391 --- /dev/null +++ b/regression-test/data/compaction/test_compaction.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 2 31 19 \N \N +2 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 2 32 20 \N \N +3 2017-10-01 Beijing 10 1 \N 2020-01-04T00:00 2020-01-05T00:00 3 34 20 \N \N +4 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 34 20 \N \N + +-- !select_default2 -- +1 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 2 31 19 \N \N +2 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 2 32 20 \N \N +3 2017-10-01 Beijing 10 1 \N 2020-01-04T00:00 2020-01-05T00:00 3 34 20 \N \N +4 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 34 20 \N \N + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index 675ca676fbf47f..f8cdfd5960e896 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -43,6 +43,8 @@ class Config { public String feHttpUser public String feHttpPassword + public String beHttpAddress + public String suitePath public String dataPath public String pluginPath @@ -77,7 +79,7 @@ class Config { Config() {} Config(String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword, - String feHttpAddress, String feHttpUser, String feHttpPassword, + String feHttpAddress, String feHttpUser, String feHttpPassword, String beHttpAddress, String suitePath, String dataPath, String testGroups, String excludeGroups, String testSuites, String excludeSuites, String testDirectories, String excludeDirectories, String pluginPath) { @@ -88,6 +90,7 @@ class Config { this.feHttpAddress = feHttpAddress this.feHttpUser = feHttpUser this.feHttpPassword = feHttpPassword + this.beHttpAddress = beHttpAddress this.suitePath = suitePath this.dataPath = dataPath this.testGroups = testGroups @@ -149,6 +152,7 @@ class Config { .toSet() config.feHttpAddress = cmd.getOptionValue(feHttpAddressOpt, config.feHttpAddress) + config.beHttpAddress = cmd.getOptionValue(beHttpAddressOpt, config.beHttpAddress) try { Inet4Address host = Inet4Address.getByName(config.feHttpAddress.split(":")[0]) as Inet4Address int port = Integer.valueOf(config.feHttpAddress.split(":")[1]) @@ -190,6 +194,7 @@ class Config { configToString(obj.feHttpAddress), configToString(obj.feHttpUser), configToString(obj.feHttpPassword), + configToString(obj.beHttpAddress), configToString(obj.suitePath), configToString(obj.dataPath), configToString(obj.testGroups), @@ -240,6 +245,11 @@ class Config { log.info("Set feHttpAddress to '${config.feHttpAddress}' because not specify.".toString()) } + if (config.beHttpAddress == null) { + config.beHttpAddress = "127.0.0.1:8040" + log.info("Set beHttpAddress to '${config.beHttpAddress}' because not specify.".toString()) + } + if (config.feHttpUser == null) { config.feHttpUser = "root" log.info("Set feHttpUser to '${config.feHttpUser}' because not specify.".toString()) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy index 3a25cfafa9b9e9..10c1b6ddef8c35 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy @@ -35,6 +35,7 @@ class ConfigOptions { static Option feHttpAddressOpt static Option feHttpUserOpt static Option feHttpPasswordOpt + static Option beHttpAddressOpt static Option pathOpt static Option dataOpt static Option pluginOpt @@ -204,6 +205,14 @@ class ConfigOptions { .longOpt("feHttpPassword") .desc("the password of fe http server") .build() + beHttpAddressOpt = Option.builder("ba") + .argName("beAddress") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("beHttpAddress") + .desc("the be http address, format is ip:port") + .build() genOutOpt = Option.builder("genOut") .required(false) .hasArg(false) @@ -287,6 +296,7 @@ class ConfigOptions { .addOption(feHttpAddressOpt) .addOption(feHttpUserOpt) .addOption(feHttpPasswordOpt) + .addOption(beHttpAddressOpt) .addOption(genOutOpt) .addOption(confFileOpt) .addOption(forceGenOutOpt) diff --git a/regression-test/suites/compaction/test_compaction.groovy b/regression-test/suites/compaction/test_compaction.groovy new file mode 100644 index 00000000000000..4722fcf5f09842 --- /dev/null +++ b/regression-test/suites/compaction/test_compaction.groovy @@ -0,0 +1,187 @@ +import org.codehaus.groovy.runtime.IOGroovyMethods + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_compaction") { + def tableName = "compaction_regression_test" + + try { + StringBuilder showConfigCommand = new StringBuilder(); + showConfigCommand.append("curl -X GET http://") + showConfigCommand.append(context.config.beHttpAddress) + showConfigCommand.append("/api/show_config") + def process = showConfigCommand.toString().execute() + int code = process.waitFor() + String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + String out = process.getText() + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + int cumulativeCompactionSkipWindowSeconds = -1 + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "cumulative_compaction_skip_window_seconds") { + cumulativeCompactionSkipWindowSeconds = Integer.parseInt(((List) ele)[2]) + } else if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME REPLACE_IF_NOT_NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME REPLACE NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间", + `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", + `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列" ) + AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20, hll_hash(1), to_bitmap(1)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19, hll_hash(2), to_bitmap(2)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21, hll_hash(2), to_bitmap(2)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(3), to_bitmap(3)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 22, hll_hash(3), to_bitmap(3)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-04', '2020-01-04', '2020-01-04', 1, 33, 21, hll_hash(4), to_bitmap(4)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 20, hll_hash(5), to_bitmap(5)) + """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 20, hll_hash(5), to_bitmap(5)) + """ + + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + String[][] tablets = sql """ show tablets from ${tableName}; """ + + if (cumulativeCompactionSkipWindowSeconds > 0) { + logger.info("Config `cumulative_compaction_skip_window_seconds` is set to " + cumulativeCompactionSkipWindowSeconds + " seconds so sleep for a while.") + Thread.sleep(cumulativeCompactionSkipWindowSeconds * 1000) + } + + // trigger compactions for all tablets in ${tableName} + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X POST http://") + sb.append(context.config.beHttpAddress) + sb.append("/api/compaction/run?tablet_id=") + sb.append(tablet_id) + sb.append("&compact_type=cumulative") + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactJson = parseJson(out.trim()) + if (compactJson.status.toLowerCase() == "fail") { + assertEquals(disableAutoCompaction, false) + logger.info("Compaction was done automatically!") + } + if (disableAutoCompaction) { + assertEquals("success", compactJson.status.toLowerCase()) + } + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(context.config.beHttpAddress) + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://") + sb.append(context.config.beHttpAddress) + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + String command = sb.toString() + // wait for cleaning stale_rowsets + process = command.execute() + code = process.waitFor() + err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + out = process.getText() + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + assert (rowCount < 8) + qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + } finally { + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +}