[feature](schema change) add schema change regression-test.

be/src/olap/schema_change.cpp

@@ -1768,7 +1768,7 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams
  rs_reader->version(), VISIBLE,
  rs_reader->rowset()->rowset_meta()->segments_overlap(),
  &new_tablet->tablet_schema(), &rowset_writer);
- if (!Status) {
+ if (!status) {
  res = Status::OLAPInternalError(OLAP_ERR_ROWSET_BUILDER_INIT);
  return process_alter_exit();
  }

regression-test/suites/schema_change/test_agg_mv_schema_change.groovy

@@ -0,0 +1,214 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("test_agg_mv_schema_change") {
+ def tableName = "schema_change_agg_mv_regression_test"
+
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE ${tableName} (
+ `user_id` LARGEINT NOT NULL COMMENT "用户id",
+ `date` DATE NOT NULL COMMENT "数据灌入日期时间",
+ `city` VARCHAR(20) COMMENT "用户所在城市",
+ `age` SMALLINT COMMENT "用户年龄",
+ `sex` TINYINT COMMENT "用户性别",
+
+ `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费",
+ `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间",
+ `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间",
+ `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列",
+ `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列")
+ AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
+ PROPERTIES ( "replication_num" = "1" );
+ """
+
+ //add materialized view
+ def result = "null"
+ def mvName = "mv1"
+ sql "create materialized view ${mvName} as select user_id, date, city, age, sex, sum(cost) from ${tableName} group by user_id, date, city, age, sex, cost;"
+ while (!result.contains("FINISHED")){
+ result = sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;"
+ result = result.toString()
+ logger.info("result: ${result}")
+ if(result.contains("CANCELLED")){
+ break
+ }
+ Thread.sleep(1000)
+ }
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (1, '2017-10-01', 'Beijing', 10, 1, 1, 30, 20, hll_hash(1), to_bitmap(1))
+ """
+ sql """ INSERT INTO ${tableName} VALUES
+ (1, '2017-10-01', 'Beijing', 10, 1, 1, 31, 19, hll_hash(2), to_bitmap(2))
+ """
+ sql """ INSERT INTO ${tableName} VALUES
+ (2, '2017-10-01', 'Beijing', 10, 1, 1, 31, 21, hll_hash(2), to_bitmap(2))
+ """
+ sql """ INSERT INTO ${tableName} VALUES
+ (2, '2017-10-01', 'Beijing', 10, 1, 1, 32, 20, hll_hash(3), to_bitmap(3))
+ """
+
+ result = "null";
+ result = sql """ select * from ${tableName} """
+ assertTrue(result.size() == 2)
+ assertTrue(result[0].size() == 10)
+ assertTrue(result[0][5] == 2, "user id 1 cost should be 2")
+ assertTrue(result[1][5] == 2, "user id 2 cost should be 2")
+ assertTrue(result[0].size() == 10)
+
+ // drop value column with mv, not light schema change
+ sql """
+ ALTER TABLE ${tableName} DROP COLUMN cost
+ """
+
+ result = "null"
+ while (!result.contains("FINISHED")){
+ result = sql "SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1;"
+ result = result.toString()
+ logger.info("result: ${result}")
+ if(result.contains("CANCELLED")) {
+ log.info("rollup job is cancelled, result: ${result}".toString())
+ break
+ }
+ Thread.sleep(1000)
+ }
+
+ sql """ INSERT INTO ${tableName} (`user_id`, `date`, `city`, `age`, `sex`, `max_dwell_time`,`min_dwell_time`, `hll_col`, `bitmap_col`)
+ VALUES
+ (3, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(4), to_bitmap(4))
+ """
+ result = "null"
+ result = sql """ SELECT * FROM ${tableName} WHERE user_id = 3 """
+
+ assertTrue(result.size() == 1)
+ assertTrue(result[0].size() == 9)
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (5, '2017-10-01', 'Beijing', 10, 1, 32, 20, hll_hash(5), to_bitmap(5))
+ """
+
+ Thread.sleep(30 * 1000)
+ // compaction
+ String[][] tablets = sql """ show tablets from ${tableName}; """
+ for (String[] tablet in tablets) {
+ String tablet_id = tablet[0]
+ logger.info("run compaction:" + tablet_id)
+ StringBuilder sb = new StringBuilder();
+ sb.append("curl -X POST http://")
+ sb.append(context.config.beHttpAddress)
+ sb.append("/api/compaction/run?tablet_id=")
+ sb.append(tablet_id)
+ sb.append("&compact_type=cumulative")
+
+ String command = sb.toString()
+ process = command.execute()
+ code = process.waitFor()
+ err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+ out = process.getText()
+ logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+ //assertEquals(code, 0)
+ }
+
+ // wait for all compactions done
+ for (String[] tablet in tablets) {
+ boolean running = true
+ do {
+ Thread.sleep(1000)
+ String tablet_id = tablet[0]
+ StringBuilder sb = new StringBuilder();
+ sb.append("curl -X GET http://")
+ sb.append(context.config.beHttpAddress)
+ sb.append("/api/compaction/run_status?tablet_id=")
+ sb.append(tablet_id)
+
+ String command = sb.toString()
+ process = command.execute()
+ code = process.waitFor()
+ err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+ out = process.getText()
+ logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+ assertEquals(code, 0)
+ def compactionStatus = parseJson(out.trim())
+ assertEquals("success", compactionStatus.status.toLowerCase())
+ running = compactionStatus.run_status
+ } while (running)
+ }
+
+ result = sql """ select count(*) from ${tableName} """
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == 4)
+
+ result = sql """ SELECT * FROM ${tableName} WHERE user_id=2 """
+ assertTrue(result.size() == 1)
+ assertTrue(result[0].size() == 9)
+
+ int rowCount = 0
+ for (String[] tablet in tablets) {
+ String tablet_id = tablet[0]
+ StringBuilder sb = new StringBuilder();
+ sb.append("curl -X GET http://")
+ sb.append(context.config.beHttpAddress)
+ sb.append("/api/compaction/show?tablet_id=")
+ sb.append(tablet_id)
+ String command = sb.toString()
+ // wait for cleaning stale_rowsets
+ process = command.execute()
+ code = process.waitFor()
+ err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+ out = process.getText()
+ logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+ assertEquals(code, 0)
+ def tabletJson = parseJson(out.trim())
+ assert tabletJson.rowsets instanceof List
+ for (String rowset in (List<String>) tabletJson.rowsets) {
+ rowCount += Integer.parseInt(rowset.split(" ")[1])
+ }
+ }
+ logger.info("size:" + rowCount)
+ assertTrue(rowCount <= 14)
+ } finally {
+ //try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+
+}
+
+