Skip to content

Commit

Permalink
fix regression case (apache#107)
Browse files Browse the repository at this point in the history
tmp
  • Loading branch information
Lchangliang committed Jun 17, 2022
1 parent 7da843b commit fc68f6c
Show file tree
Hide file tree
Showing 11 changed files with 166 additions and 76 deletions.
6 changes: 4 additions & 2 deletions be/src/exec/olap_scanner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include "gen_cpp/PaloInternalService_types.h"
#include "olap/decimal12.h"
#include "olap/field.h"
#include "olap/tablet_schema.h"
#include "olap/uint24.h"
#include "olap_scan_node.h"
#include "olap_utils.h"
Expand Down Expand Up @@ -249,8 +250,9 @@ Status OlapScanner::_init_return_columns() {
if (!slot->is_materialized()) {
continue;
}
int32_t index = slot->col_unique_id() >= 0 ? slot->col_unique_id()
: _tablet_schema.field_index(slot->col_name());
int32_t index = slot->col_unique_id() >= 0
? _tablet_schema.field_index(slot->col_unique_id())
: _tablet_schema.field_index(slot->col_name());
if (index < 0) {
std::stringstream ss;
ss << "field name is invalid. field=" << slot->col_name();
Expand Down
3 changes: 2 additions & 1 deletion be/src/olap/tablet.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include <filesystem>
#include <map>
#include <set>
#include <shared_mutex>

#include "olap/base_compaction.h"
#include "olap/cumulative_compaction.h"
Expand Down Expand Up @@ -1656,7 +1657,7 @@ std::shared_ptr<MemTracker>& Tablet::get_compaction_mem_tracker(CompactionType c
}

const TabletSchema& Tablet::tablet_schema() const {
std::lock_guard<std::shared_mutex> wrlock(_meta_lock);
std::shared_lock wrlock(_meta_lock);
const RowsetSharedPtr last_rowset = rowset_with_max_version();
if (last_rowset == nullptr) {
return _schema;
Expand Down
7 changes: 7 additions & 0 deletions be/src/olap/tablet_schema.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -442,12 +442,14 @@ void TabletSchema::append_column(TabletColumn column) {
_num_null_columns++;
}
_field_name_to_index[column.name()] = _num_columns;
_field_id_to_index[column.col_unique_id()] = _num_columns;
_cols.push_back(std::move(column));
_num_columns++;
}

void TabletSchema::clear_columns() {
_field_name_to_index.clear();
_field_id_to_index.clear();
_num_columns = 0;
_num_null_columns = 0;
_num_key_columns = 0;
Expand Down Expand Up @@ -590,6 +592,11 @@ int32_t TabletSchema::field_index(const std::string& field_name) const {
return (found == _field_name_to_index.end()) ? -1 : found->second;
}

int32_t TabletSchema::field_index(int32_t col_unique_od) const {
const auto& found = _field_id_to_index.find(col_unique_od);
return (found == _field_id_to_index.end()) ? -1 : found->second;
}

const std::vector<TabletColumn>& TabletSchema::columns() const {
return _cols;
}
Expand Down
2 changes: 2 additions & 0 deletions be/src/olap/tablet_schema.h
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ class TabletSchema {

size_t row_size() const;
int32_t field_index(const std::string& field_name) const;
int32_t field_index(int32_t col_unique_id) const;
const TabletColumn& column(size_t ordinal) const;
const std::vector<TabletColumn>& columns() const;
size_t num_columns() const { return _num_columns; }
Expand Down Expand Up @@ -184,6 +185,7 @@ class TabletSchema {
size_t _sort_col_num = 0;
std::vector<TabletColumn> _cols;
std::unordered_map<std::string, int32_t> _field_name_to_index;
std::unordered_map<int32_t, int32_t> _field_id_to_index;
size_t _num_columns = 0;
size_t _num_key_columns = 0;
size_t _num_null_columns = 0;
Expand Down
8 changes: 7 additions & 1 deletion be/src/olap/tuple_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,13 @@ Status TupleReader::init(const ReaderParams& read_params) {
if (!status.ok()) {
return status;
}


if (_optimize_for_single_rowset(rs_readers)) {
_next_row_func = _tablet->keys_type() == AGG_KEYS ? &TupleReader::_direct_agg_key_next_row
: &TupleReader::_direct_next_row;
return Status::OK();
}

switch (_tablet->keys_type()) {
case KeysType::DUP_KEYS:
_next_row_func = &TupleReader::_direct_next_row;
Expand Down
63 changes: 0 additions & 63 deletions regression-test/suites/query/aggregate/window_funnel.groovy

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ suite ("test_agg_keys_schema_change") {
logger.info("run compaction:" + tablet_id)
StringBuilder sb = new StringBuilder();
sb.append("curl -X POST http://")
sb.append("192.168.0.21:8041")
sb.append(context.config.beHttpAddress)
sb.append("/api/compaction/run?tablet_id=")
sb.append(tablet_id)
sb.append("&compact_type=cumulative")
Expand All @@ -172,7 +172,7 @@ suite ("test_agg_keys_schema_change") {
String tablet_id = tablet[0]
StringBuilder sb = new StringBuilder();
sb.append("curl -X GET http://")
sb.append("192.168.0.21:8041")
sb.append(context.config.beHttpAddress)
sb.append("/api/compaction/run_status?tablet_id=")
sb.append(tablet_id)

Expand Down Expand Up @@ -201,7 +201,7 @@ suite ("test_agg_keys_schema_change") {
String tablet_id = tablet[0]
StringBuilder sb = new StringBuilder();
sb.append("curl -X GET http://")
sb.append("192.168.0.21:8041")
sb.append(context.config.beHttpAddress)
sb.append("/api/compaction/show?tablet_id=")
sb.append(tablet_id)
String command = sb.toString()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ suite ("test_dup_keys_schema_change") {
logger.info("run compaction:" + tablet_id)
StringBuilder sb = new StringBuilder();
sb.append("curl -X POST http://")
sb.append("192.168.0.21:8041")
sb.append(context.config.beHttpAddress)
sb.append("/api/compaction/run?tablet_id=")
sb.append(tablet_id)
sb.append("&compact_type=cumulative")
Expand All @@ -162,7 +162,7 @@ suite ("test_dup_keys_schema_change") {
String tablet_id = tablet[0]
StringBuilder sb = new StringBuilder();
sb.append("curl -X GET http://")
sb.append("192.168.0.21:8041")
sb.append(context.config.beHttpAddress)
sb.append("/api/compaction/run_status?tablet_id=")
sb.append(tablet_id)

Expand Down Expand Up @@ -192,7 +192,7 @@ suite ("test_dup_keys_schema_change") {
String tablet_id = tablet[0]
StringBuilder sb = new StringBuilder();
sb.append("curl -X GET http://")
sb.append("192.168.0.21:8041")
sb.append(context.config.beHttpAddress)
sb.append("/api/compaction/show?tablet_id=")
sb.append(tablet_id)
String command = sb.toString()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ suite ("test_uniq_keys_schema_change") {
logger.info("run compaction:" + tablet_id)
StringBuilder sb = new StringBuilder();
sb.append("curl -X POST http://")
sb.append("192.168.0.21:8041")
sb.append(context.config.beHttpAddress)
sb.append("/api/compaction/run?tablet_id=")
sb.append(tablet_id)
sb.append("&compact_type=cumulative")
Expand All @@ -162,7 +162,7 @@ suite ("test_uniq_keys_schema_change") {
String tablet_id = tablet[0]
StringBuilder sb = new StringBuilder();
sb.append("curl -X GET http://")
sb.append("192.168.0.21:8041")
sb.append(context.config.beHttpAddress)
sb.append("/api/compaction/run_status?tablet_id=")
sb.append(tablet_id)

Expand Down Expand Up @@ -191,7 +191,7 @@ suite ("test_uniq_keys_schema_change") {
String tablet_id = tablet[0]
StringBuilder sb = new StringBuilder();
sb.append("curl -X GET http://")
sb.append("192.168.0.21:8041")
sb.append(context.config.beHttpAddress)
sb.append("/api/compaction/show?tablet_id=")
sb.append(tablet_id)
String command = sb.toString()
Expand Down
86 changes: 86 additions & 0 deletions regression-test/suites/table_function/explode_json_array.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// The cases is copied from
// https://spark.apache.org/docs/latest/sql-ref-syntax-qry-select-lateral-view.html
// and modified by Doris.

suite("explode_json_array") {
def tableName = "person"

sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE ${tableName}
(id INT, name STRING, age INT, class INT, address STRING)
UNIQUE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 8
PROPERTIES("replication_num" = "1")
"""

sql """ INSERT INTO ${tableName} VALUES
(100, 'John', 30, 1, 'Street 1'),
(200, 'Mary', NULL, 1, 'Street 2'),
(300, 'Mike', 80, 3, 'Street 3'),
(400, 'Dan', 50, 4, 'Street 4') """

// not vectorized
qt_explode_json_array """ SELECT * FROM ${tableName}
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[30, 60]') t1 as c_age
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[40, 80]') t2 as d_age
ORDER BY id, c_age, d_age """

qt_explode_json_array """ SELECT c_age, COUNT(1) FROM ${tableName}
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[30, 60]') t1 as c_age
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[40, 80]') t2 as d_age
GROUP BY c_age ORDER BY c_age """

qt_explode_json_array """ SELECT * FROM ${tableName}
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[]') t1 AS c_age
ORDER BY id, c_age """

qt_explode_json_array """ SELECT * FROM ${tableName}
LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[1, "b", 3]') t1 as c
LATERAL VIEW EXPLODE_JSON_ARRAY_DOUBLE('[1.23, 22.214, 214.1]') t2 as d
ORDER BY id, c, d """

qt_outer_join_explode_json_array """SELECT id, age, e1 FROM (SELECT id, age, e1 FROM (SELECT b.id, a.age FROM
${tableName} a LEFT JOIN ${tableName} b ON a.id=b.age)T LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[1, "b", 3]')
TMP AS e1) AS T ORDER BY age, e1"""

// vectorized
sql """ set enable_vectorized_engine = true """

qt_explode_json_array """ select @@enable_vectorized_engine """
qt_explode_json_array """ SELECT * FROM ${tableName}
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[30, 60]') t1 as c_age
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[40, 80]') t2 as d_age
ORDER BY id, c_age, d_age """

qt_explode_json_array """ SELECT c_age, COUNT(1) FROM ${tableName}
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[30, 60]') t1 as c_age
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[40, 80]') t2 as d_age
GROUP BY c_age ORDER BY c_age """

qt_explode_json_array """ SELECT * FROM ${tableName}
LATERAL VIEW EXPLODE_JSON_ARRAY_INT('[]') t1 AS c_age
ORDER BY id, c_age """

qt_explode_json_array """ SELECT * FROM ${tableName}
LATERAL VIEW EXPLODE_JSON_ARRAY_STRING('[1, "b", 3]') t1 as c
LATERAL VIEW EXPLODE_JSON_ARRAY_DOUBLE('[1.23, 22.214, 214.1]') t2 as d
ORDER BY id, c, d """

}
49 changes: 49 additions & 0 deletions regression-test/suites/table_function/explode_split.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("explode_split") {
def tableName = "test_lv_str"

sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE ${tableName}
(k1 INT, k2 STRING)
UNIQUE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 8
PROPERTIES("replication_num" = "1")
"""

sql """ INSERT INTO ${tableName} VALUES (1, 'a,b,c') """

// not_vectorized
qt_explode_split """ select * from ${tableName}
lateral view explode_split(k2, ',') tmp1 as e1 """

qt_explode_split """ select * from ${tableName}
lateral view explode_split(k2, ',') tmp1 as e1
lateral view explode_split(k2, ',') tmp2 as e2 """

// vectorized
sql """ set enable_vectorized_engine = true """

qt_explode_split """ select * from ${tableName}
lateral view explode_split(k2, ',') tmp1 as e1 """

qt_explode_split """ select * from ${tableName}
lateral view explode_split(k2, ',') tmp1 as e1
lateral view explode_split(k2, ',') tmp2 as e2 """

}

0 comments on commit fc68f6c

Please sign in to comment.