diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index 2ea0c827074c76..adac8365d61cb9 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -24,6 +24,7 @@ #include "gen_cpp/PaloInternalService_types.h" #include "olap/decimal12.h" #include "olap/field.h" +#include "olap/tablet_schema.h" #include "olap/uint24.h" #include "olap_scan_node.h" #include "olap_utils.h" @@ -249,8 +250,9 @@ Status OlapScanner::_init_return_columns() { if (!slot->is_materialized()) { continue; } - int32_t index = slot->col_unique_id() >= 0 ? slot->col_unique_id() - : _tablet_schema.field_index(slot->col_name()); + int32_t index = slot->col_unique_id() >= 0 + ? _tablet_schema.field_index(slot->col_unique_id()) + : _tablet_schema.field_index(slot->col_name()); if (index < 0) { std::stringstream ss; ss << "field name is invalid. field=" << slot->col_name(); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 21b7ff2e35e8dd..c2275ff69de446 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include "olap/base_compaction.h" #include "olap/cumulative_compaction.h" @@ -1656,7 +1657,7 @@ std::shared_ptr& Tablet::get_compaction_mem_tracker(CompactionType c } const TabletSchema& Tablet::tablet_schema() const { - std::lock_guard wrlock(_meta_lock); + std::shared_lock wrlock(_meta_lock); const RowsetSharedPtr last_rowset = rowset_with_max_version(); if (last_rowset == nullptr) { return _schema; diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 20079ae3a90a53..028710b7ca5b0c 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -442,12 +442,14 @@ void TabletSchema::append_column(TabletColumn column) { _num_null_columns++; } _field_name_to_index[column.name()] = _num_columns; + _field_id_to_index[column.col_unique_id()] = _num_columns; _cols.push_back(std::move(column)); _num_columns++; } void TabletSchema::clear_columns() { _field_name_to_index.clear(); + _field_id_to_index.clear(); _num_columns = 0; _num_null_columns = 0; _num_key_columns = 0; @@ -590,6 +592,11 @@ int32_t TabletSchema::field_index(const std::string& field_name) const { return (found == _field_name_to_index.end()) ? -1 : found->second; } +int32_t TabletSchema::field_index(int32_t col_unique_od) const { + const auto& found = _field_id_to_index.find(col_unique_od); + return (found == _field_id_to_index.end()) ? -1 : found->second; +} + const std::vector& TabletSchema::columns() const { return _cols; } diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index a2d452f03f3031..6535f3a76c08db 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -140,6 +140,7 @@ class TabletSchema { size_t row_size() const; int32_t field_index(const std::string& field_name) const; + int32_t field_index(int32_t col_unique_id) const; const TabletColumn& column(size_t ordinal) const; const std::vector& columns() const; size_t num_columns() const { return _num_columns; } @@ -184,6 +185,7 @@ class TabletSchema { size_t _sort_col_num = 0; std::vector _cols; std::unordered_map _field_name_to_index; + std::unordered_map _field_id_to_index; size_t _num_columns = 0; size_t _num_key_columns = 0; size_t _num_null_columns = 0; diff --git a/be/src/olap/tuple_reader.cpp b/be/src/olap/tuple_reader.cpp index fd83d32a5e24a3..b7a50b68088476 100644 --- a/be/src/olap/tuple_reader.cpp +++ b/be/src/olap/tuple_reader.cpp @@ -73,7 +73,13 @@ Status TupleReader::init(const ReaderParams& read_params) { if (!status.ok()) { return status; } - + + if (_optimize_for_single_rowset(rs_readers)) { + _next_row_func = _tablet->keys_type() == AGG_KEYS ? &TupleReader::_direct_agg_key_next_row + : &TupleReader::_direct_next_row; + return Status::OK(); + } + switch (_tablet->keys_type()) { case KeysType::DUP_KEYS: _next_row_func = &TupleReader::_direct_next_row; diff --git a/regression-test/suites/query/aggregate/window_funnel.groovy b/regression-test/suites/query/aggregate/window_funnel.groovy deleted file mode 100644 index 279b433b53ddbb..00000000000000 --- a/regression-test/suites/query/aggregate/window_funnel.groovy +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/aggregate -// and modified by Doris. - -suite("window_funnel") { - def tableName = "windowfunnel_test" - - sql """ DROP TABLE IF EXISTS ${tableName} """ - sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - xwho varchar(50) NULL COMMENT 'xwho', - xwhen datetime COMMENT 'xwhen', - xwhat int NULL COMMENT 'xwhat' - ) - DUPLICATE KEY(xwho) - DISTRIBUTED BY HASH(xwho) BUCKETS 3 - PROPERTIES ( - "replication_num" = "1" - ); - """ - sql "INSERT into ${tableName} (xwho, xwhen, xwhat) VALUES('1', '2022-03-12 10:41:00', 1)" - sql "INSERT INTO ${tableName} (xwho, xwhen, xwhat) VALUES('1', '2022-03-12 13:28:02', 2)" - sql "INSERT INTO ${tableName} (xwho, xwhen, xwhat) VALUES('1', '2022-03-12 16:15:01', 3)" - sql "INSERT INTO ${tableName} (xwho, xwhen, xwhat) VALUES('1', '2022-03-12 19:05:04', 4)" - - qt_window_funnel """ select - window_funnel( - 1, - 'default', - t.xwhen, - t.xwhat = 1, - t.xwhat = 2 - ) AS level - from ${tableName} t; - """ - qt_window_funnel """ select - window_funnel( - 20000, - 'default', - t.xwhen, - t.xwhat = 1, - t.xwhat = 2 - ) AS level - from ${tableName} t; - """ -} diff --git a/regression-test/suites/schema_change/test_agg_keys_schema_change.groovy b/regression-test/suites/schema_change/test_agg_keys_schema_change.groovy index 9d0abaeae73e85..e368bf43509d96 100644 --- a/regression-test/suites/schema_change/test_agg_keys_schema_change.groovy +++ b/regression-test/suites/schema_change/test_agg_keys_schema_change.groovy @@ -150,7 +150,7 @@ suite ("test_agg_keys_schema_change") { logger.info("run compaction:" + tablet_id) StringBuilder sb = new StringBuilder(); sb.append("curl -X POST http://") - sb.append("192.168.0.21:8041") + sb.append(context.config.beHttpAddress) sb.append("/api/compaction/run?tablet_id=") sb.append(tablet_id) sb.append("&compact_type=cumulative") @@ -172,7 +172,7 @@ suite ("test_agg_keys_schema_change") { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); sb.append("curl -X GET http://") - sb.append("192.168.0.21:8041") + sb.append(context.config.beHttpAddress) sb.append("/api/compaction/run_status?tablet_id=") sb.append(tablet_id) @@ -201,7 +201,7 @@ suite ("test_agg_keys_schema_change") { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); sb.append("curl -X GET http://") - sb.append("192.168.0.21:8041") + sb.append(context.config.beHttpAddress) sb.append("/api/compaction/show?tablet_id=") sb.append(tablet_id) String command = sb.toString() diff --git a/regression-test/suites/schema_change/test_dup_keys_schema_change.groovy b/regression-test/suites/schema_change/test_dup_keys_schema_change.groovy index f21494ef4e5408..ca7d264a5b1a49 100644 --- a/regression-test/suites/schema_change/test_dup_keys_schema_change.groovy +++ b/regression-test/suites/schema_change/test_dup_keys_schema_change.groovy @@ -140,7 +140,7 @@ suite ("test_dup_keys_schema_change") { logger.info("run compaction:" + tablet_id) StringBuilder sb = new StringBuilder(); sb.append("curl -X POST http://") - sb.append("192.168.0.21:8041") + sb.append(context.config.beHttpAddress) sb.append("/api/compaction/run?tablet_id=") sb.append(tablet_id) sb.append("&compact_type=cumulative") @@ -162,7 +162,7 @@ suite ("test_dup_keys_schema_change") { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); sb.append("curl -X GET http://") - sb.append("192.168.0.21:8041") + sb.append(context.config.beHttpAddress) sb.append("/api/compaction/run_status?tablet_id=") sb.append(tablet_id) @@ -192,7 +192,7 @@ suite ("test_dup_keys_schema_change") { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); sb.append("curl -X GET http://") - sb.append("192.168.0.21:8041") + sb.append(context.config.beHttpAddress) sb.append("/api/compaction/show?tablet_id=") sb.append(tablet_id) String command = sb.toString() diff --git a/regression-test/suites/schema_change/test_uniq_keys_schema_change.groovy b/regression-test/suites/schema_change/test_uniq_keys_schema_change.groovy index 46ee2208be4ea4..2f4e6c3f860ad1 100644 --- a/regression-test/suites/schema_change/test_uniq_keys_schema_change.groovy +++ b/regression-test/suites/schema_change/test_uniq_keys_schema_change.groovy @@ -140,7 +140,7 @@ suite ("test_uniq_keys_schema_change") { logger.info("run compaction:" + tablet_id) StringBuilder sb = new StringBuilder(); sb.append("curl -X POST http://") - sb.append("192.168.0.21:8041") + sb.append(context.config.beHttpAddress) sb.append("/api/compaction/run?tablet_id=") sb.append(tablet_id) sb.append("&compact_type=cumulative") @@ -162,7 +162,7 @@ suite ("test_uniq_keys_schema_change") { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); sb.append("curl -X GET http://") - sb.append("192.168.0.21:8041") + sb.append(context.config.beHttpAddress) sb.append("/api/compaction/run_status?tablet_id=") sb.append(tablet_id) @@ -191,7 +191,7 @@ suite ("test_uniq_keys_schema_change") { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); sb.append("curl -X GET http://") - sb.append("192.168.0.21:8041") + sb.append(context.config.beHttpAddress) sb.append("/api/compaction/show?tablet_id=") sb.append(tablet_id) String command = sb.toString() diff --git a/regression-test/suites/table_function/explode_json_array.groovy b/regression-test/suites/table_function/explode_json_array.groovy new file mode 100644 index 00000000000000..c1d4a3cf992943 --- /dev/null +++ b/regression-test/suites/table_function/explode_json_array.groovy @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The cases is copied from +// https://spark.apache.org/docs/latest/sql-ref-syntax-qry-select-lateral-view.html +// and modified by Doris. + +suite("explode_json_array") { + def tableName = "person" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} + (id INT, name STRING, age INT, class INT, address STRING) + UNIQUE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 8 + PROPERTIES("replication_num" = "1") + """ + + sql """ INSERT INTO ${tableName} VALUES + (100, 'John', 30, 1, 'Street 1'), + (200, 'Mary', NULL, 1, 'Street 2'), + (300, 'Mike', 80, 3, 'Street 3'), + (400, 'Dan', 50, 4, 'Street 4') """ + + // not vectorized + qt_explode_json_array """ SELECT * FROM ${tableName} + LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[30, 60]') t1 as c_age + LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[40, 80]') t2 as d_age + ORDER BY id, c_age, d_age """ + + qt_explode_json_array """ SELECT c_age, COUNT(1) FROM ${tableName} + LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[30, 60]') t1 as c_age + LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[40, 80]') t2 as d_age + GROUP BY c_age ORDER BY c_age """ + + qt_explode_json_array """ SELECT * FROM ${tableName} + LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[]') t1 AS c_age + ORDER BY id, c_age """ + + qt_explode_json_array """ SELECT * FROM ${tableName} + LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[1, "b", 3]') t1 as c + LATERAL VIEW EXPLODE_JSON_ARRAY_DOUBLE('[1.23, 22.214, 214.1]') t2 as d + ORDER BY id, c, d """ + + qt_outer_join_explode_json_array """SELECT id, age, e1 FROM (SELECT id, age, e1 FROM (SELECT b.id, a.age FROM + ${tableName} a LEFT JOIN ${tableName} b ON a.id=b.age)T LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[1, "b", 3]') + TMP AS e1) AS T ORDER BY age, e1""" + + // vectorized + sql """ set enable_vectorized_engine = true """ + + qt_explode_json_array """ select @@enable_vectorized_engine """ + qt_explode_json_array """ SELECT * FROM ${tableName} + LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[30, 60]') t1 as c_age + LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[40, 80]') t2 as d_age + ORDER BY id, c_age, d_age """ + + qt_explode_json_array """ SELECT c_age, COUNT(1) FROM ${tableName} + LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[30, 60]') t1 as c_age + LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[40, 80]') t2 as d_age + GROUP BY c_age ORDER BY c_age """ + + qt_explode_json_array """ SELECT * FROM ${tableName} + LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[]') t1 AS c_age + ORDER BY id, c_age """ + + qt_explode_json_array """ SELECT * FROM ${tableName} + LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[1, "b", 3]') t1 as c + LATERAL VIEW EXPLODE_JSON_ARRAY_DOUBLE('[1.23, 22.214, 214.1]') t2 as d + ORDER BY id, c, d """ + +} \ No newline at end of file diff --git a/regression-test/suites/table_function/explode_split.groovy b/regression-test/suites/table_function/explode_split.groovy new file mode 100644 index 00000000000000..b12099e691754d --- /dev/null +++ b/regression-test/suites/table_function/explode_split.groovy @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("explode_split") { + def tableName = "test_lv_str" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} + (k1 INT, k2 STRING) + UNIQUE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 8 + PROPERTIES("replication_num" = "1") + """ + + sql """ INSERT INTO ${tableName} VALUES (1, 'a,b,c') """ + + // not_vectorized + qt_explode_split """ select * from ${tableName} + lateral view explode_split(k2, ',') tmp1 as e1 """ + + qt_explode_split """ select * from ${tableName} + lateral view explode_split(k2, ',') tmp1 as e1 + lateral view explode_split(k2, ',') tmp2 as e2 """ + + // vectorized + sql """ set enable_vectorized_engine = true """ + + qt_explode_split """ select * from ${tableName} + lateral view explode_split(k2, ',') tmp1 as e1 """ + + qt_explode_split """ select * from ${tableName} + lateral view explode_split(k2, ',') tmp1 as e1 + lateral view explode_split(k2, ',') tmp2 as e2 """ + +} \ No newline at end of file