diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index ac51aed9e80b7d..db7dd9e1ce920e 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -513,6 +513,9 @@ void TaskWorkerPool::_alter_tablet(TaskWorkerPool* worker_pool_this, &error_msgs, process_name); OLAPStatus sc_status = worker_pool_this->_env->storage_engine()->execute_task(&engine_task); if (sc_status != OLAP_SUCCESS) { + if (sc_status == OLAP_ERR_DATA_QUALITY_ERR) { + error_msgs.push_back("The data quality does not satisfy, please check your data. "); + } status = DORIS_ERROR; } else { status = DORIS_SUCCESS; diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h index b3c136b1ddf1d7..d726d7f7a5777b 100644 --- a/be/src/olap/olap_define.h +++ b/be/src/olap/olap_define.h @@ -305,6 +305,8 @@ enum OLAPStatus { OLAP_ERR_PREVIOUS_SCHEMA_CHANGE_NOT_FINISHED = -1603, OLAP_ERR_SCHEMA_CHANGE_INFO_INVALID = -1604, OLAP_ERR_QUERY_SPLIT_KEY_ERR = -1605, + //Error caused by a data quality issue during schema change/materialized view + OLAP_ERR_DATA_QUALITY_ERR = -1606, // Column File // [-1700, -1800) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 5e84aa59577066..b8426213292b44 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -176,7 +176,7 @@ ColumnMapping* RowBlockChanger::get_mutable_column_mapping(size_t column_index) << " origin_type=" \ << ref_block->tablet_schema().column(ref_column).type() \ << ", alter_type=" << mutable_block->tablet_schema().column(i).type(); \ - return false; \ + return OLAP_ERR_SCHEMA_CHANGE_INFO_INVALID; \ } \ break; \ } @@ -385,23 +385,23 @@ bool count_field(RowCursor* read_helper, RowCursor* write_helper, const TabletCo return true; } -bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_version, +OLAPStatus RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_version, RowBlock* mutable_block, uint64_t* filtered_rows) const { if (mutable_block == nullptr) { LOG(FATAL) << "mutable block is uninitialized."; - return false; + return OLAP_ERR_NOT_INITED; } else if (mutable_block->tablet_schema().num_columns() != _schema_mapping.size()) { LOG(WARNING) << "mutable block does not match with schema mapping rules. " << "block_schema_size=" << mutable_block->tablet_schema().num_columns() << ", mapping_schema_size=" << _schema_mapping.size(); - return false; + return OLAP_ERR_NOT_INITED; } if (mutable_block->capacity() < ref_block->row_block_info().row_num) { LOG(WARNING) << "mutable block is not large enough for storing the changed block. " << "mutable_block_size=" << mutable_block->capacity() << ", ref_block_row_num=" << ref_block->row_block_info().row_num; - return false; + return OLAP_ERR_NOT_INITED; } mutable_block->clear(); @@ -409,13 +409,13 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v RowCursor write_helper; if (write_helper.init(mutable_block->tablet_schema()) != OLAP_SUCCESS) { LOG(WARNING) << "fail to init rowcursor."; - return false; + return OLAP_ERR_NOT_INITED; } RowCursor read_helper; if (read_helper.init(ref_block->tablet_schema()) != OLAP_SUCCESS) { LOG(WARNING) << "fail to init rowcursor."; - return false; + return OLAP_ERR_NOT_INITED; } // a.1 先判断数据是否需要过滤,最终只有标记为1的才是留下需要的 @@ -465,7 +465,7 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v _do_materialized_transform = count_field; } else { LOG(WARNING) << "error materialized view function : " << _schema_mapping[i].materialized_function; - return false; + return OLAP_ERR_SCHEMA_CHANGE_INFO_INVALID; } VLOG(3) << "_schema_mapping[" << i << "].materialized_function : " << _schema_mapping[i].materialized_function; for (size_t row_index = 0, new_row_index = 0; @@ -479,7 +479,7 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v if (!_do_materialized_transform(&read_helper, &write_helper, ref_block->tablet_schema().column(ref_column), i, _schema_mapping[i].ref_column, mem_pool)) { - return false; + return OLAP_ERR_DATA_QUALITY_ERR; } } continue; @@ -578,7 +578,7 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v << "the column type which was altered from was unsupported." << "status:" << st << ", from_type=" << reftype << ", to_type=" << newtype; - return false; + return st; } } } @@ -607,7 +607,7 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v LOG(WARNING) << "the column type which was altered from was unsupported." << " from_type=" << ref_block->tablet_schema().column(ref_column).type(); - return false; + return OLAP_ERR_SCHEMA_CHANGE_INFO_INVALID; } if (newtype < reftype) { @@ -644,7 +644,7 @@ bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_v // (其实在init时就可以重新init成少的,filter留下的new_row_num) // 在split_table时,可能会出现因为过滤导致没有数据 mutable_block->finalize(new_row_num); - return true; + return OLAP_SUCCESS; } #undef CONVERT_FROM_TYPE @@ -912,7 +912,7 @@ bool RowBlockMerger::_pop_heap() { return true; } -bool LinkedSchemaChange::process(RowsetReaderSharedPtr rowset_reader, +OLAPStatus LinkedSchemaChange::process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) { OLAPStatus status = new_rowset_writer->add_rowset_for_linked_schema_change( @@ -923,10 +923,8 @@ bool LinkedSchemaChange::process(RowsetReaderSharedPtr rowset_reader, << ", base_tablet=" << base_tablet->full_name() << ", version=" << new_rowset_writer->version().first << "-" << new_rowset_writer->version().second; - return false; } - - return true; + return status; } SchemaChangeDirectly::SchemaChangeDirectly(const RowBlockChanger& row_block_changer) @@ -950,13 +948,13 @@ bool SchemaChangeDirectly::_write_row_block(RowsetWriter* rowset_writer, RowBloc return true; } -bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer, +OLAPStatus SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer, TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) { if (_row_block_allocator == nullptr) { _row_block_allocator = new RowBlockAllocator(new_tablet->tablet_schema(), 0); if (_row_block_allocator == nullptr) { LOG(FATAL) << "failed to malloc RowBlockAllocator. size=" << sizeof(RowBlockAllocator); - return false; + return OLAP_ERR_INPUT_PARAMETER_ERROR; } } @@ -964,12 +962,12 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr _cursor = new (nothrow) RowCursor(); if (nullptr == _cursor) { LOG(WARNING) << "fail to allocate row cursor."; - return false; + return OLAP_ERR_INPUT_PARAMETER_ERROR; } if (OLAP_SUCCESS != _cursor->init(new_tablet->tablet_schema())) { LOG(WARNING) << "fail to init row cursor."; - return false; + return OLAP_ERR_INPUT_PARAMETER_ERROR; } } @@ -991,14 +989,13 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr LOG(WARNING) << "create empty version for schema change failed." << "version=" << rowset_writer->version().first << "-" << rowset_writer->version().second; - return false; + return OLAP_ERR_INPUT_PARAMETER_ERROR; } - return true; + return OLAP_SUCCESS; } VLOG(3) << "init writer. new_tablet=" << new_tablet->full_name() << ", block_row_number=" << new_tablet->num_rows_per_row_block(); - bool result = true; RowBlock* new_row_block = nullptr; // Reset filtered_rows and merged_rows statistic @@ -1015,12 +1012,10 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr _row_block_allocator->release(new_row_block); new_row_block = nullptr; } - - if (OLAP_SUCCESS != - _row_block_allocator->allocate(&new_row_block, - ref_row_block->row_block_info().row_num, true)) { + res = _row_block_allocator->allocate(&new_row_block, + ref_row_block->row_block_info().row_num, true); + if (OLAP_SUCCESS != res) { LOG(WARNING) << "failed to allocate RowBlock."; - result = false; goto DIRECTLY_PROCESS_ERR; } } else { @@ -1029,10 +1024,9 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr // 将ref改为new。这一步按道理来说确实需要等大的块,但理论上和writer无关。 uint64_t filtered_rows = 0; - if (!_row_block_changer.change_row_block(ref_row_block, rowset_reader->version().second, - new_row_block, &filtered_rows)) { + if ((res = _row_block_changer.change_row_block(ref_row_block, rowset_reader->version().second, + new_row_block, &filtered_rows)) != OLAP_SUCCESS) { LOG(WARNING) << "failed to change data in row block."; - result = false; goto DIRECTLY_PROCESS_ERR; } // rows filtered by delete handler one by one @@ -1040,7 +1034,7 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr if (!_write_row_block(rowset_writer, new_row_block)) { LOG(WARNING) << "failed to write row block."; - result = false; + res = OLAP_ERR_SCHEMA_CHANGE_INFO_INVALID; goto DIRECTLY_PROCESS_ERR; } @@ -1049,7 +1043,7 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr } if (OLAP_SUCCESS != rowset_writer->flush()) { - result = false; + res = OLAP_ERR_ALTER_STATUS_ERR; goto DIRECTLY_PROCESS_ERR; } @@ -1065,7 +1059,7 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr << ", merged_rows=" << merged_rows() << ", filtered_rows=" << filtered_rows() << ", new_index_rows=" << rowset_writer->num_rows(); - result = false; + res = OLAP_ERR_ALTER_STATUS_ERR; } LOG(INFO) << "all row nums. source_rows=" << rowset_reader->rowset()->num_rows() << ", merged_rows=" << merged_rows() << ", filtered_rows=" << filtered_rows() @@ -1081,7 +1075,7 @@ bool SchemaChangeDirectly::process(RowsetReaderSharedPtr rowset_reader, RowsetWr _row_block_allocator->release(new_row_block); new_row_block = nullptr; } - return result; + return res; } SchemaChangeWithSorting::SchemaChangeWithSorting(const RowBlockChanger& row_block_changer, @@ -1103,7 +1097,7 @@ SchemaChangeWithSorting::~SchemaChangeWithSorting() { SAFE_DELETE(_row_block_allocator); } -bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, +OLAPStatus SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) { if (_row_block_allocator == nullptr) { @@ -1111,7 +1105,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, new (nothrow) RowBlockAllocator(new_tablet->tablet_schema(), _memory_limitation); if (_row_block_allocator == nullptr) { LOG(FATAL) << "failed to malloc RowBlockAllocator. size=" << sizeof(RowBlockAllocator); - return false; + return OLAP_ERR_INPUT_PARAMETER_ERROR; } } @@ -1128,12 +1122,11 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, LOG(WARNING) << "create empty version for schema change failed." << " version=" << new_rowset_writer->version().first << "-" << new_rowset_writer->version().second; - return false; + return OLAP_ERR_INPUT_PARAMETER_ERROR; } - return true; + return OLAP_SUCCESS; } - bool result = true; RowBlockSorter row_block_sorter(_row_block_allocator); // for internal sorting @@ -1163,7 +1156,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, ref_row_block->row_block_info().row_num, true)) { LOG(WARNING) << "failed to allocate RowBlock."; - result = false; + res = OLAP_ERR_INPUT_PARAMETER_ERROR; goto SORTING_PROCESS_ERR; } @@ -1171,7 +1164,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, if (row_block_arr.size() < 1) { LOG(WARNING) << "Memory limitation is too small for Schema Change." << "memory_limitation=" << _memory_limitation; - return false; + return OLAP_ERR_INPUT_PARAMETER_ERROR; } // enter here while memory limitation is reached. @@ -1186,7 +1179,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, rowset_reader->version_hash(), new_tablet, new_rowset_type, segments_overlap, &rowset)) { LOG(WARNING) << "failed to sorting internally."; - result = false; + res = OLAP_ERR_ALTER_STATUS_ERR; goto SORTING_PROCESS_ERR; } @@ -1208,7 +1201,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, if (!_row_block_changer.change_row_block(ref_row_block, rowset_reader->version().second, new_row_block, &filtered_rows)) { LOG(WARNING) << "failed to change data in row block."; - result = false; + res = OLAP_ERR_ALTER_STATUS_ERR; goto SORTING_PROCESS_ERR; } add_filtered_rows(filtered_rows); @@ -1216,7 +1209,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, if (new_row_block->row_block_info().row_num > 0) { if (!row_block_sorter.sort(&new_row_block)) { LOG(WARNING) << "failed to sort row block."; - result = false; + res = OLAP_ERR_ALTER_STATUS_ERR; OLAP_GOTO(SORTING_PROCESS_ERR); } @@ -1244,7 +1237,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, rowset_reader->version_hash(), new_tablet, new_rowset_type, segments_overlap, &rowset)) { LOG(WARNING) << "failed to sorting internally."; - result = false; + res = OLAP_ERR_ALTER_STATUS_ERR; goto SORTING_PROCESS_ERR; } @@ -1267,11 +1260,11 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, LOG(WARNING) << "create empty version for schema change failed." << " version=" << new_rowset_writer->version().first << "-" << new_rowset_writer->version().second; - return false; + return OLAP_ERR_ALTER_STATUS_ERR; } } else if (!_external_sorting(src_rowsets, new_rowset_writer, new_tablet)) { LOG(WARNING) << "failed to sorting externally."; - result = false; + res = OLAP_ERR_ALTER_STATUS_ERR; goto SORTING_PROCESS_ERR; } @@ -1286,7 +1279,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, << ", merged_rows=" << merged_rows() << ", filtered_rows=" << filtered_rows() << ", new_index_rows=" << new_rowset_writer->num_rows(); - result = false; + res = OLAP_ERR_ALTER_STATUS_ERR; } LOG(INFO) << "all row nums. source_rows=" << rowset_reader->rowset()->num_rows() << ", merged_rows=" << merged_rows() << ", filtered_rows=" << filtered_rows() @@ -1310,7 +1303,7 @@ bool SchemaChangeWithSorting::process(RowsetReaderSharedPtr rowset_reader, } row_block_arr.clear(); - return result; + return res; } bool SchemaChangeWithSorting::_internal_sorting(const vector& row_block_arr, @@ -1722,7 +1715,8 @@ OLAPStatus SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tabl std::unique_ptr rowset_writer; RowsetFactory::create_rowset_writer(writer_context, &rowset_writer); - if (!sc_procedure->process(rowset_reader, rowset_writer.get(), new_tablet, base_tablet)) { + if ((res = sc_procedure->process(rowset_reader, rowset_writer.get(), new_tablet, base_tablet)) != OLAP_SUCCESS) { + if ((*base_rowset)->is_pending()) { LOG(WARNING) << "failed to process the transaction when schema change. " << "tablet=" << new_tablet->full_name() << "'" @@ -1732,7 +1726,6 @@ OLAPStatus SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tabl << "version=" << (*base_rowset)->version().first << "-" << (*base_rowset)->version().second; } - res = OLAP_ERR_INPUT_PARAMETER_ERROR; new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + rowset_writer->rowset_id().to_string()); goto SCHEMA_VERSION_CONVERT_ERR; @@ -1930,12 +1923,11 @@ OLAPStatus SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangePa goto PROCESS_ALTER_EXIT; } - if (!sc_procedure->process(rs_reader, rowset_writer.get(), sc_params.new_tablet, - sc_params.base_tablet)) { + if ((res = sc_procedure->process(rs_reader, rowset_writer.get(), sc_params.new_tablet, + sc_params.base_tablet)) != OLAP_SUCCESS) { LOG(WARNING) << "failed to process the version." << " version=" << rs_reader->version().first << "-" << rs_reader->version().second; - res = OLAP_ERR_INPUT_PARAMETER_ERROR; new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + rowset_writer->rowset_id().to_string()); goto PROCESS_ALTER_EXIT; diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h index 7771ff7a515c15..6ce83dd14e860a 100644 --- a/be/src/olap/schema_change.h +++ b/be/src/olap/schema_change.h @@ -59,7 +59,7 @@ class RowBlockChanger { SchemaMapping get_schema_mapping() const { return _schema_mapping; } - bool change_row_block(const RowBlock* ref_block, int32_t data_version, RowBlock* mutable_block, + OLAPStatus change_row_block(const RowBlock* ref_block, int32_t data_version, RowBlock* mutable_block, uint64_t* filtered_rows) const; private: @@ -92,7 +92,7 @@ class SchemaChange { SchemaChange() : _filtered_rows(0), _merged_rows(0) {} virtual ~SchemaChange() {} - virtual bool process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_builder, + virtual OLAPStatus process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_builder, TabletSharedPtr tablet, TabletSharedPtr base_tablet) = 0; void add_filtered_rows(uint64_t filtered_rows) { _filtered_rows += filtered_rows; } @@ -118,7 +118,7 @@ class LinkedSchemaChange : public SchemaChange { : _row_block_changer(row_block_changer) {} ~LinkedSchemaChange() {} - bool process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer, + virtual OLAPStatus process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) override; private: @@ -134,7 +134,7 @@ class SchemaChangeDirectly : public SchemaChange { explicit SchemaChangeDirectly(const RowBlockChanger& row_block_changer); virtual ~SchemaChangeDirectly(); - virtual bool process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer, + virtual OLAPStatus process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) override; private: @@ -154,7 +154,7 @@ class SchemaChangeWithSorting : public SchemaChange { size_t memory_limitation); virtual ~SchemaChangeWithSorting(); - virtual bool process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_builder, + virtual OLAPStatus process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_builder, TabletSharedPtr new_tablet, TabletSharedPtr base_tablet) override; private: diff --git a/be/test/olap/schema_change_test.cpp b/be/test/olap/schema_change_test.cpp index 8367fa8bbaf653..e7ede79bf83d4f 100644 --- a/be/test/olap/schema_change_test.cpp +++ b/be/test/olap/schema_change_test.cpp @@ -781,7 +781,6 @@ TEST_F(TestColumn, ConvertIntToBitmap) { column_mapping->ref_column = 2; column_mapping->materialized_function = "to_bitmap"; - RowBlock mutable_block(&mv_tablet_schema); mutable_block.init(block_info); uint64_t filtered_rows = 0; @@ -878,6 +877,86 @@ TEST_F(TestColumn, ConvertCharToHLL) { HyperLogLog hll(*dst_slice); ASSERT_EQ(hll.estimate_cardinality(), 1); } + +TEST_F(TestColumn, ConvertCharToCount) { + //Base Tablet + TabletSchema tablet_schema; + CreateTabletSchema(tablet_schema); + + //Base row block + CreateColumnWriter(tablet_schema); + + RowBlock block(&tablet_schema); + RowBlockInfo block_info; + block_info.row_num = 10000; + block.init(block_info); + + RowCursor write_row; + write_row.init(tablet_schema); + write_row.allocate_memory_for_string_type(tablet_schema); + std::vector val_string_array; + val_string_array.emplace_back("1"); + val_string_array.emplace_back("1"); + val_string_array.emplace_back("2"); + val_string_array.emplace_back("3"); + OlapTuple tuple(val_string_array); + write_row.from_tuple(tuple); + block.set_row(0, write_row); + + block.finalize(1); + ColumnDataHeaderMessage header; + ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); + + //Materialized View tablet schema + TabletSchemaPB mv_tablet_schema_pb; + mv_tablet_schema_pb.set_keys_type(KeysType::AGG_KEYS); + mv_tablet_schema_pb.set_num_short_key_columns(2); + mv_tablet_schema_pb.set_num_rows_per_row_block(1024); + mv_tablet_schema_pb.set_compress_kind(COMPRESS_NONE); + mv_tablet_schema_pb.set_next_column_unique_id(3); + + ColumnPB* mv_column_1 = mv_tablet_schema_pb.add_column(); + mv_column_1->set_unique_id(1); + mv_column_1->set_name("k1"); + mv_column_1->set_type("INT"); + mv_column_1->set_is_key(true); + mv_column_1->set_length(4); + mv_column_1->set_index_length(4); + mv_column_1->set_is_nullable(false); + mv_column_1->set_is_bf_column(false); + + ColumnPB* mv_column_2 = mv_tablet_schema_pb.add_column(); + mv_column_2->set_unique_id(2); + mv_column_2->set_name("v1"); + mv_column_2->set_type("BIGINT"); + mv_column_2->set_length(4); + mv_column_2->set_is_key(false); + mv_column_2->set_is_nullable(false); + mv_column_2->set_is_bf_column(false); + mv_column_2->set_aggregation("SUM"); + + TabletSchema mv_tablet_schema; + mv_tablet_schema.init_from_pb(mv_tablet_schema_pb); + + RowBlockChanger row_block_changer(mv_tablet_schema); + ColumnMapping* column_mapping = row_block_changer.get_mutable_column_mapping(0); + column_mapping->ref_column = 0; + column_mapping = row_block_changer.get_mutable_column_mapping(1); + column_mapping->ref_column = 1; + column_mapping->materialized_function = "count_field"; + + RowBlock mutable_block(&mv_tablet_schema); + mutable_block.init(block_info); + uint64_t filtered_rows = 0; + row_block_changer.change_row_block(&block, 0, &mutable_block, &filtered_rows); + + RowCursor mv_row_cursor; + mv_row_cursor.init(mv_tablet_schema); + mutable_block.get_row(0, &mv_row_cursor); + + auto dst = mv_row_cursor.cell_ptr(1); + ASSERT_EQ(*(int64_t *)dst, 1); +} } int main(int argc, char** argv) { diff --git a/docs/en/administrator-guide/materialized_view.md b/docs/en/administrator-guide/materialized_view.md index e79f275d0308d0..6e8c016d97dbb2 100644 --- a/docs/en/administrator-guide/materialized_view.md +++ b/docs/en/administrator-guide/materialized_view.md @@ -483,3 +483,8 @@ This problem can be solved by creating a materialized view with k3 as the first 2. If the conditional column of the delete statement does not exist in the materialized view, the delete operation cannot be performed. If you must delete data, you need to delete the materialized view before deleting the data. 3. Too many materialized views on a single table will affect the efficiency of importing: When importing data, the materialized view and base table data are updated synchronously. If a table has more than 10 materialized view tables, it may cause the import speed to be very high. slow. This is the same as a single import needs to import 10 tables at the same time. 4. The same column with different aggregate functions cannot appear in a materialized view at the same time. For example, select sum(a), min(a) from table are not supported. + +## Error +1. DATA_QUALITY_ERR: "The data quality does not satisfy, please check your data" +Materialized view creation failed due to data quality issues. +Note: The bitmap type only supports positive integers. If there are negative Numbers in the original data, the materialized view will fail to be created \ No newline at end of file diff --git a/docs/zh-CN/administrator-guide/materialized_view.md b/docs/zh-CN/administrator-guide/materialized_view.md index a0fe43c4d72650..ae79fe6b284d9d 100644 --- a/docs/zh-CN/administrator-guide/materialized_view.md +++ b/docs/zh-CN/administrator-guide/materialized_view.md @@ -485,3 +485,8 @@ MySQL [test]> desc advertiser_view_record; 2. 如果删除语句的条件列,在物化视图中不存在,则不能进行删除操作。如果一定要删除数据,则需要先将物化视图删除,然后方可删除数据。 3. 单表上过多的物化视图会影响导入的效率:导入数据时,物化视图和 base 表数据是同步更新的,如果一张表的物化视图表超过10张,则有可能导致导入速度很慢。这就像单次导入需要同时导入10张表数据是一样的。 4. 相同列,不同聚合函数,不能同时出现在一张物化视图中,比如:select sum(a), min(a) from table 不支持。 + +## 异常错误 +1. DATA_QUALITY_ERR: "The data quality does not satisfy, please check your data" + 由于数据质量问题导致物化视图创建失败。 + 注意:bitmap类型仅支持正整型, 如果原始数据中存在负数,会导致物化视图创建失败 diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java index 46a90edb88cb71..d0406d231dbaca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java @@ -135,6 +135,7 @@ public TMasterResult finishTask(TFinishTaskRequest request) { if (taskStatus.getStatus_code() != TStatusCode.OK) { task.failed(); String errMsg = "task type: " + taskType + ", status_code: " + taskStatus.getStatus_code().toString() + + (taskStatus.isSetError_msgs() ? (", status_message: " + taskStatus.getError_msgs()) : "") + ", backendId: " + backend + ", signature: " + signature; task.setErrorMsg(errMsg); // We start to let FE perceive the task's error msg