From cacef8d04df841dc0e4ae6f540ccca6ff808bf05 Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Mon, 19 Aug 2024 11:01:49 +0800 Subject: [PATCH 1/7] [refactor](loadmemlimit) remove load memlimit since it is never used (#39536) It is needed to pick to branch 21, because we will depend on it to do spill disk. Co-authored-by: yiguolei --- be/src/runtime/runtime_state.cpp | 29 +++++++++++++++---- be/src/runtime/runtime_state.h | 4 --- be/src/vec/sink/writer/vtablet_writer.cpp | 2 -- be/src/vec/sink/writer/vtablet_writer.h | 3 -- .../doris/load/loadv2/LoadLoadingTask.java | 8 ----- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 2713ee441dd0df..82c1198e9332ea 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -463,13 +463,30 @@ Status RuntimeState::append_error_msg_to_file(std::function line, return Status::OK(); } -int64_t RuntimeState::get_load_mem_limit() { - // TODO: the code is abandoned, it can be deleted after v1.3 - if (_query_options.__isset.load_mem_limit && _query_options.load_mem_limit > 0) { - return _query_options.load_mem_limit; - } else { - return _query_mem_tracker->limit(); +std::string RuntimeState::get_error_log_file_path() { + if (_s3_error_fs && _error_log_file && _error_log_file->is_open()) { + // close error log file + _error_log_file->close(); + std::string error_log_absolute_path = + _exec_env->load_path_mgr()->get_load_error_absolute_path(_error_log_file_path); + // upload error log file to s3 + Status st = _s3_error_fs->upload(error_log_absolute_path, _s3_error_log_file_path); + if (st.ok()) { + // remove local error log file + std::filesystem::remove(error_log_absolute_path); + } else { + // upload failed and return local error log file path + LOG(WARNING) << "Fail to upload error file to s3, error_log_file_path=" + << _error_log_file_path << ", error=" << st; + return _error_log_file_path; + } + // expiration must be less than a week (in seconds) for presigned url + static const unsigned EXPIRATION_SECONDS = 7 * 24 * 60 * 60 - 1; + // We should return a public endpoint to user. + _error_log_file_path = _s3_error_fs->generate_presigned_url(_s3_error_log_file_path, + EXPIRATION_SECONDS, true); } + return _error_log_file_path; } void RuntimeState::resize_op_id_to_local_state(int operator_size) { diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 8b8cbd85f0f117..8243faa37aeec5 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -466,10 +466,6 @@ class RuntimeState { std::vector& error_tablet_infos() { return _error_tablet_infos; } - // get mem limit for load channel - // if load mem limit is not set, or is zero, using query mem limit instead. - int64_t get_load_mem_limit(); - // local runtime filter mgr, the runtime filter do not have remote target or // not need local merge should regist here. the instance exec finish, the local // runtime filter mgr can release the memory of local runtime filter diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp index 1e6b8f7b8687b6..d1651bb3a92419 100644 --- a/be/src/vec/sink/writer/vtablet_writer.cpp +++ b/be/src/vec/sink/writer/vtablet_writer.cpp @@ -413,7 +413,6 @@ void VNodeChannel::_open_internal(bool is_incremental) { request->set_num_senders(_parent->_num_senders); request->set_need_gen_rollup(false); // Useless but it is a required field in pb - request->set_load_mem_limit(_parent->_load_mem_limit); request->set_load_channel_timeout_s(_parent->_load_channel_timeout_s); request->set_is_high_priority(_parent->_is_high_priority); request->set_sender_ip(BackendOptions::get_localhost()); @@ -1226,7 +1225,6 @@ Status VTabletWriter::_init(RuntimeState* state, RuntimeProfile* profile) { _max_wait_exec_timer = ADD_TIMER(profile, "MaxWaitExecTime"); _add_batch_number = ADD_COUNTER(profile, "NumberBatchAdded", TUnit::UNIT); _num_node_channels = ADD_COUNTER(profile, "NumberNodeChannels", TUnit::UNIT); - _load_mem_limit = state->get_load_mem_limit(); #ifdef DEBUG // check: tablet ids should be unique diff --git a/be/src/vec/sink/writer/vtablet_writer.h b/be/src/vec/sink/writer/vtablet_writer.h index 603034cea6d7a5..ba986fbc6d4e0f 100644 --- a/be/src/vec/sink/writer/vtablet_writer.h +++ b/be/src/vec/sink/writer/vtablet_writer.h @@ -660,9 +660,6 @@ class VTabletWriter final : public AsyncResultWriter { RuntimeProfile::Counter* _add_batch_number = nullptr; RuntimeProfile::Counter* _num_node_channels = nullptr; - // load mem limit is for remote load channel - int64_t _load_mem_limit = -1; - // the timeout of load channels opened by this tablet sink. in second int64_t _load_channel_timeout_s = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java index d60c17233d7a08..14cd742be45cd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java @@ -152,14 +152,6 @@ private void executeOnce() throws Exception { curCoordinator.setExecMemoryLimit(execMemLimit); curCoordinator.setExecPipEngine(Config.enable_pipeline_load); - /* - * For broker load job, user only need to set mem limit by 'exec_mem_limit' property. - * And the variable 'load_mem_limit' does not make any effect. - * However, in order to ensure the consistency of semantics when executing on the BE side, - * and to prevent subsequent modification from incorrectly setting the load_mem_limit, - * here we use exec_mem_limit to directly override the load_mem_limit property. - */ - curCoordinator.setLoadMemLimit(execMemLimit); curCoordinator.setMemTableOnSinkNode(enableMemTableOnSinkNode); long leftTimeMs = getLeftTimeMs(); From 9c036a0720b04b19644c949df9aa6718164f5d93 Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Tue, 27 Aug 2024 09:45:52 +0800 Subject: [PATCH 2/7] [enhancement](memory) add exception handle logic in memtable flush (#39913) ## Proposed changes Issue Number: close #xxx --------- Co-authored-by: yiguolei --- be/src/olap/memtable.cpp | 7 ++++++- be/src/olap/memtable.h | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 923849162db331..a4df4b8f6742aa 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -505,7 +505,7 @@ bool MemTable::need_agg() const { return false; } -Status MemTable::to_block(std::unique_ptr* res) { +Status MemTable::_to_block(std::unique_ptr* res) { size_t same_keys_num = _sort(); if (_keys_type == KeysType::DUP_KEYS || same_keys_num == 0) { if (_keys_type == KeysType::DUP_KEYS && _tablet_schema->num_key_columns() == 0) { @@ -529,4 +529,9 @@ Status MemTable::to_block(std::unique_ptr* res) { return Status::OK(); } +Status MemTable::to_block(std::unique_ptr* res) { + RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_to_block(res)); + return Status::OK(); +} + } // namespace doris diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index 916067ba1193d2..70f7a9f22a0aa8 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -205,6 +205,9 @@ class MemTable { void _aggregate_two_row_in_block(vectorized::MutableBlock& mutable_block, RowInBlock* new_row, RowInBlock* row_in_skiplist); + // Used to wrapped by to_block to do exception handle logic + Status _to_block(std::unique_ptr* res); + private: int64_t _tablet_id; bool _enable_unique_key_mow = false; From 18dffc1d336a0be56f3139683725133103fc59f9 Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Tue, 27 Aug 2024 20:45:49 +0800 Subject: [PATCH 3/7] [enhancement](memory) return error if allocate memory failed during page builder allocate memory failed (#39964) ## Proposed changes Issue Number: close #xxx --------- Co-authored-by: yiguolei --- .../rowset/segment_v2/binary_dict_page.cpp | 14 ++++++----- .../olap/rowset/segment_v2/binary_dict_page.h | 2 +- .../rowset/segment_v2/binary_plain_page.h | 25 +++++++++++-------- .../rowset/segment_v2/binary_prefix_page.cpp | 23 +++++++++-------- .../rowset/segment_v2/binary_prefix_page.h | 2 +- .../olap/rowset/segment_v2/bitshuffle_page.h | 5 ++-- .../olap/rowset/segment_v2/column_writer.cpp | 10 +++++--- .../segment_v2/frame_of_reference_page.h | 5 ++-- .../segment_v2/indexed_column_writer.cpp | 3 ++- be/src/olap/rowset/segment_v2/page_builder.h | 3 ++- be/src/olap/rowset/segment_v2/plain_page.h | 18 +++++++------ be/src/olap/rowset/segment_v2/rle_page.h | 5 ++-- 12 files changed, 67 insertions(+), 48 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp index 52795f0338a79f..8270adfbde8bf1 100644 --- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp +++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp @@ -142,7 +142,7 @@ Status BinaryDictPageBuilder::add(const uint8_t* vals, size_t* count) { } } -OwnedSlice BinaryDictPageBuilder::finish() { +Status BinaryDictPageBuilder::finish(OwnedSlice* slice) { if (VLOG_DEBUG_IS_ON && _encoding_type == DICT_ENCODING) { VLOG_DEBUG << "dict page size:" << _dict_builder->size(); } @@ -150,11 +150,14 @@ OwnedSlice BinaryDictPageBuilder::finish() { DCHECK(!_finished); _finished = true; - OwnedSlice data_slice = _data_page_builder->finish(); + OwnedSlice data_slice; + RETURN_IF_ERROR(_data_page_builder->finish(&data_slice)); // TODO(gaodayue) separate page header and content to avoid this copy - _buffer.append(data_slice.slice().data, data_slice.slice().size); + RETURN_IF_CATCH_EXCEPTION( + { _buffer.append(data_slice.slice().data, data_slice.slice().size); }); encode_fixed32_le(&_buffer[0], _encoding_type); - return _buffer.build(); + *slice = _buffer.build(); + return Status::OK(); } Status BinaryDictPageBuilder::reset() { @@ -183,8 +186,7 @@ uint64_t BinaryDictPageBuilder::size() const { } Status BinaryDictPageBuilder::get_dictionary_page(OwnedSlice* dictionary_page) { - *dictionary_page = _dict_builder->finish(); - return Status::OK(); + return _dict_builder->finish(dictionary_page); } Status BinaryDictPageBuilder::get_first_value(void* value) const { diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.h b/be/src/olap/rowset/segment_v2/binary_dict_page.h index 2a8467e7def516..d069eb9f7edc98 100644 --- a/be/src/olap/rowset/segment_v2/binary_dict_page.h +++ b/be/src/olap/rowset/segment_v2/binary_dict_page.h @@ -68,7 +68,7 @@ class BinaryDictPageBuilder : public PageBuilderHelper { Status add(const uint8_t* vals, size_t* count) override; - OwnedSlice finish() override; + Status finish(OwnedSlice* slice) override; Status reset() override; diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h b/be/src/olap/rowset/segment_v2/binary_plain_page.h index 3fe76c5d3aee84..69d79fbcc5b3e3 100644 --- a/be/src/olap/rowset/segment_v2/binary_plain_page.h +++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h @@ -93,19 +93,22 @@ class BinaryPlainPageBuilder : public PageBuilderHelper 0) { - _copy_value_at(0, &_first_value); - _copy_value_at(_offsets.size() - 1, &_last_value); - } - return _buffer.build(); + RETURN_IF_CATCH_EXCEPTION({ + // Set up trailer + for (uint32_t _offset : _offsets) { + put_fixed32_le(&_buffer, _offset); + } + put_fixed32_le(&_buffer, _offsets.size()); + if (_offsets.size() > 0) { + _copy_value_at(0, &_first_value); + _copy_value_at(_offsets.size() - 1, &_last_value); + } + *slice = _buffer.build(); + }); + return Status::OK(); } Status reset() override { diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp index 9d1ecdb9470778..34eb14951aeb32 100644 --- a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp +++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp @@ -88,18 +88,21 @@ Status BinaryPrefixPageBuilder::add(const uint8_t* vals, size_t* add_count) { return Status::OK(); } -OwnedSlice BinaryPrefixPageBuilder::finish() { +Status BinaryPrefixPageBuilder::finish(OwnedSlice* slice) { DCHECK(!_finished); _finished = true; - put_fixed32_le(&_buffer, (uint32_t)_count); - uint8_t restart_point_internal = RESTART_POINT_INTERVAL; - _buffer.append(&restart_point_internal, 1); - auto restart_point_size = _restart_points_offset.size(); - for (uint32_t i = 0; i < restart_point_size; ++i) { - put_fixed32_le(&_buffer, _restart_points_offset[i]); - } - put_fixed32_le(&_buffer, restart_point_size); - return _buffer.build(); + RETURN_IF_CATCH_EXCEPTION({ + put_fixed32_le(&_buffer, (uint32_t)_count); + uint8_t restart_point_internal = RESTART_POINT_INTERVAL; + _buffer.append(&restart_point_internal, 1); + auto restart_point_size = _restart_points_offset.size(); + for (uint32_t i = 0; i < restart_point_size; ++i) { + put_fixed32_le(&_buffer, _restart_points_offset[i]); + } + put_fixed32_le(&_buffer, restart_point_size); + *slice = _buffer.build(); + }); + return Status::OK(); } const uint8_t* BinaryPrefixPageDecoder::_decode_value_lengths(const uint8_t* ptr, uint32_t* shared, diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.h b/be/src/olap/rowset/segment_v2/binary_prefix_page.h index de4ec60070bad6..41deb4e6c1fe1a 100644 --- a/be/src/olap/rowset/segment_v2/binary_prefix_page.h +++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.h @@ -52,7 +52,7 @@ class BinaryPrefixPageBuilder : public PageBuilderHelper 0) { _first_value = cell(0); _last_value = cell(_count - 1); } - return _finish(SIZE_OF_TYPE); + RETURN_IF_CATCH_EXCEPTION({ *slice = _finish(SIZE_OF_TYPE); }); + return Status::OK(); } Status reset() override { diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp index e463b883fd206d..bdbfcdc2d41621 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.cpp +++ b/be/src/olap/rowset/segment_v2/column_writer.cpp @@ -70,9 +70,10 @@ class NullBitmapBuilder { // Returns whether the building nullmap contains nullptr bool has_null() const { return _has_null; } - OwnedSlice finish() { + Status finish(OwnedSlice* slice) { _rle_encoder.Flush(); - return _bitmap_buf.build(); + RETURN_IF_CATCH_EXCEPTION({ *slice = _bitmap_buf.build(); }); + return Status::OK(); } void reset() { @@ -723,14 +724,15 @@ Status ScalarColumnWriter::finish_current_page() { // build data page body : encoded values + [nullmap] std::vector body; - OwnedSlice encoded_values = _page_builder->finish(); + OwnedSlice encoded_values; + RETURN_IF_ERROR(_page_builder->finish(&encoded_values)); RETURN_IF_ERROR(_page_builder->reset()); body.push_back(encoded_values.slice()); OwnedSlice nullmap; if (_null_bitmap_builder != nullptr) { if (is_nullable() && _null_bitmap_builder->has_null()) { - nullmap = _null_bitmap_builder->finish(); + RETURN_IF_ERROR(_null_bitmap_builder->finish(&nullmap)); body.push_back(nullmap.slice()); } _null_bitmap_builder->reset(); diff --git a/be/src/olap/rowset/segment_v2/frame_of_reference_page.h b/be/src/olap/rowset/segment_v2/frame_of_reference_page.h index 4477912803b3bb..5aedf126b55ce2 100644 --- a/be/src/olap/rowset/segment_v2/frame_of_reference_page.h +++ b/be/src/olap/rowset/segment_v2/frame_of_reference_page.h @@ -54,11 +54,12 @@ class FrameOfReferencePageBuilder : public PageBuilderHelperflush(); - return _buf.build(); + RETURN_IF_CATCH_EXCEPTION({ *slice = _buf.build(); }); + return Status::OK(); } Status reset() override { diff --git a/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp b/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp index e1b238084a9ce5..51606d818899ec 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp +++ b/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp @@ -117,7 +117,8 @@ Status IndexedColumnWriter::_finish_current_data_page(size_t& num_val) { ordinal_t first_ordinal = _num_values - num_values_in_page; // IndexedColumn doesn't have NULLs, thus data page body only contains encoded values - OwnedSlice page_body = _data_page_builder->finish(); + OwnedSlice page_body; + RETURN_IF_ERROR(_data_page_builder->finish(&page_body)); RETURN_IF_ERROR(_data_page_builder->reset()); PageFooterPB footer; diff --git a/be/src/olap/rowset/segment_v2/page_builder.h b/be/src/olap/rowset/segment_v2/page_builder.h index 61fa2eaf8e1d66..7e24c56796cbb5 100644 --- a/be/src/olap/rowset/segment_v2/page_builder.h +++ b/be/src/olap/rowset/segment_v2/page_builder.h @@ -63,7 +63,8 @@ class PageBuilder { // Finish building the current page, return the encoded data. // This api should be followed by reset() before reusing the builder - virtual OwnedSlice finish() = 0; + // It will return error status when memory allocated failed during finish + virtual Status finish(OwnedSlice* owned_slice) = 0; // Get the dictionary page for dictionary encoding mode column. virtual Status get_dictionary_page(OwnedSlice* dictionary_page) { diff --git a/be/src/olap/rowset/segment_v2/plain_page.h b/be/src/olap/rowset/segment_v2/plain_page.h index af31275002ad3c..28b1e96d206fbb 100644 --- a/be/src/olap/rowset/segment_v2/plain_page.h +++ b/be/src/olap/rowset/segment_v2/plain_page.h @@ -59,14 +59,18 @@ class PlainPageBuilder : public PageBuilderHelper > { return Status::OK(); } - OwnedSlice finish() override { + Status finish(OwnedSlice* slice) override { encode_fixed32_le((uint8_t*)&_buffer[0], _count); - if (_count > 0) { - _first_value.assign_copy(&_buffer[PLAIN_PAGE_HEADER_SIZE], SIZE_OF_TYPE); - _last_value.assign_copy(&_buffer[PLAIN_PAGE_HEADER_SIZE + (_count - 1) * SIZE_OF_TYPE], - SIZE_OF_TYPE); - } - return _buffer.build(); + RETURN_IF_CATCH_EXCEPTION({ + if (_count > 0) { + _first_value.assign_copy(&_buffer[PLAIN_PAGE_HEADER_SIZE], SIZE_OF_TYPE); + _last_value.assign_copy( + &_buffer[PLAIN_PAGE_HEADER_SIZE + (_count - 1) * SIZE_OF_TYPE], + SIZE_OF_TYPE); + } + *slice = _buffer.build(); + }); + return Status::OK(); } Status reset() override { diff --git a/be/src/olap/rowset/segment_v2/rle_page.h b/be/src/olap/rowset/segment_v2/rle_page.h index 40ec587743c1a2..d1974f18d39ddc 100644 --- a/be/src/olap/rowset/segment_v2/rle_page.h +++ b/be/src/olap/rowset/segment_v2/rle_page.h @@ -94,14 +94,15 @@ class RlePageBuilder : public PageBuilderHelper > { return Status::OK(); } - OwnedSlice finish() override { + Status finish(OwnedSlice* slice) override { DCHECK(!_finished); _finished = true; // here should Flush first and then encode the count header // or it will lead to a bug if the header is less than 8 byte and the data is small _rle_encoder->Flush(); encode_fixed32_le(&_buf[0], _count); - return _buf.build(); + *slice = _buf.build(); + return Status::OK(); } Status reset() override { From dac18f1ab130da400a4c005f171816cb9a88c409 Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Wed, 28 Aug 2024 21:05:11 +0800 Subject: [PATCH 4/7] [enhancement](exception) pageio method should return error when allocate memory failed (#40020) 1. pageio method should return error when allocate memory failed just for point query. 2. variant parsing logic should return error when allocate memory failed. --------- Co-authored-by: yiguolei --- be/src/olap/rowset/segment_creator.cpp | 2 +- be/src/olap/rowset/segment_creator.h | 6 +++++- be/src/olap/rowset/segment_v2/page_io.cpp | 4 ++-- be/src/olap/rowset/segment_v2/page_io.h | 11 ++++++++++- be/src/vec/common/schema_util.cpp | 15 +++++---------- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/be/src/olap/rowset/segment_creator.cpp b/be/src/olap/rowset/segment_creator.cpp index 641b32535561cc..bf10ff3f1ed880 100644 --- a/be/src/olap/rowset/segment_creator.cpp +++ b/be/src/olap/rowset/segment_creator.cpp @@ -85,7 +85,7 @@ Status SegmentFlusher::flush_single_block(const vectorized::Block* block, int32_ return Status::OK(); } -Status SegmentFlusher::_parse_variant_columns(vectorized::Block& block) { +Status SegmentFlusher::_internal_parse_variant_columns(vectorized::Block& block) { size_t num_rows = block.rows(); if (num_rows == 0) { return Status::OK(); diff --git a/be/src/olap/rowset/segment_creator.h b/be/src/olap/rowset/segment_creator.h index 93508e9629ddbb..7fa69b2c57c718 100644 --- a/be/src/olap/rowset/segment_creator.h +++ b/be/src/olap/rowset/segment_creator.h @@ -138,7 +138,11 @@ class SegmentFlusher { bool need_buffering(); private: - Status _parse_variant_columns(vectorized::Block& block); + // This method will catch exception when allocate memory failed + Status _parse_variant_columns(vectorized::Block& block) { + RETURN_IF_CATCH_EXCEPTION({ return _internal_parse_variant_columns(block); }); + } + Status _internal_parse_variant_columns(vectorized::Block& block); Status _add_rows(std::unique_ptr& segment_writer, const vectorized::Block* block, size_t row_offset, size_t row_num); Status _add_rows(std::unique_ptr& segment_writer, diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp b/be/src/olap/rowset/segment_v2/page_io.cpp index cea4a23f742178..07d5656ee8a44b 100644 --- a/be/src/olap/rowset/segment_v2/page_io.cpp +++ b/be/src/olap/rowset/segment_v2/page_io.cpp @@ -111,8 +111,8 @@ Status PageIO::write_page(io::FileWriter* writer, const std::vector& body return Status::OK(); } -Status PageIO::read_and_decompress_page(const PageReadOptions& opts, PageHandle* handle, - Slice* body, PageFooterPB* footer) { +Status PageIO::read_and_decompress_page_(const PageReadOptions& opts, PageHandle* handle, + Slice* body, PageFooterPB* footer) { opts.sanity_check(); opts.stats->total_pages_num++; diff --git a/be/src/olap/rowset/segment_v2/page_io.h b/be/src/olap/rowset/segment_v2/page_io.h index 31c81880dac650..889dae6d34efe6 100644 --- a/be/src/olap/rowset/segment_v2/page_io.h +++ b/be/src/olap/rowset/segment_v2/page_io.h @@ -123,8 +123,17 @@ class PageIO { // `handle' holds the memory of page data, // `body' points to page body, // `footer' stores the page footer. + // This method is exception safe, it will failed when allocate memory failed. static Status read_and_decompress_page(const PageReadOptions& opts, PageHandle* handle, - Slice* body, PageFooterPB* footer); + Slice* body, PageFooterPB* footer) { + RETURN_IF_CATCH_EXCEPTION( + { return read_and_decompress_page_(opts, handle, body, footer); }); + } + +private: + // An internal method that not deal with exception. + static Status read_and_decompress_page_(const PageReadOptions& opts, PageHandle* handle, + Slice* body, PageFooterPB* footer); }; } // namespace segment_v2 diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index bb7b64992dee27..4db526ed9b5ba4 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -544,16 +544,11 @@ Status _parse_variant_columns(Block& block, const std::vector& variant_pos, } Status parse_variant_columns(Block& block, const std::vector& variant_pos, - const ParseContext& ctx) { - try { - // Parse each variant column from raw string column - RETURN_IF_ERROR(vectorized::schema_util::_parse_variant_columns(block, variant_pos, ctx)); - } catch (const doris::Exception& e) { - // TODO more graceful, max_filter_ratio - LOG(WARNING) << "encounter execption " << e.to_string(); - return Status::InternalError(e.to_string()); - } - return Status::OK(); + const ParseConfig& config) { + // Parse each variant column from raw string column + RETURN_IF_CATCH_EXCEPTION({ + return vectorized::schema_util::_parse_variant_columns(block, variant_pos, config); + }); } void finalize_variant_columns(Block& block, const std::vector& variant_pos, From 73ee6a8896aa38dfbc864ed3c9b6dc18e47f3c1f Mon Sep 17 00:00:00 2001 From: hui lai <1353307710@qq.com> Date: Mon, 12 Aug 2024 11:17:43 +0800 Subject: [PATCH 5/7] [opt](bytebuffer) allocate ByteBuffer memory by Allocator and make it exception safe (#38960) At present, the memory allocation of `ByteBuffer` is done through `new char[capacity_]`. Now, it is uniformly allocated by `Allocator` for the following purposes: 1. Better memory statistics 2. Better support for memory limit check --- be/src/http/action/http_stream.cpp | 51 +++++++++++-------- be/src/http/action/stream_load.cpp | 32 ++++++++---- be/src/io/fs/stream_load_pipe.cpp | 7 ++- be/src/runtime/exec_env.h | 4 ++ be/src/runtime/exec_env_init.cpp | 2 + .../runtime/stream_load/stream_load_context.h | 8 ++- .../stream_load/stream_load_executor.cpp | 4 ++ be/src/util/byte_buffer.h | 19 +++++-- 8 files changed, 87 insertions(+), 40 deletions(-) diff --git a/be/src/http/action/http_stream.cpp b/be/src/http/action/http_stream.cpp index 7dd85653002e37..cd9b10bea975c6 100644 --- a/be/src/http/action/http_stream.cpp +++ b/be/src/http/action/http_stream.cpp @@ -234,31 +234,40 @@ void HttpStreamAction::on_chunk_data(HttpRequest* req) { struct evhttp_request* ev_req = req->get_evhttp_request(); auto evbuf = evhttp_request_get_input_buffer(ev_req); + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->stream_load_pipe_tracker()); + int64_t start_read_data_time = MonotonicNanos(); while (evbuffer_get_length(evbuf) > 0) { - auto bb = ByteBuffer::allocate(128 * 1024); - auto remove_bytes = evbuffer_remove(evbuf, bb->ptr, bb->capacity); - bb->pos = remove_bytes; - bb->flip(); - auto st = ctx->body_sink->append(bb); - // schema_buffer stores 1M of data for parsing column information - // need to determine whether to cache for the first time - if (ctx->is_read_schema) { - if (ctx->schema_buffer->pos + remove_bytes < config::stream_tvf_buffer_size) { - ctx->schema_buffer->put_bytes(bb->ptr, remove_bytes); - } else { - LOG(INFO) << "use a portion of data to request fe to obtain column information"; - ctx->is_read_schema = false; - ctx->status = process_put(req, ctx); + try { + auto bb = ByteBuffer::allocate(128 * 1024); + auto remove_bytes = evbuffer_remove(evbuf, bb->ptr, bb->capacity); + bb->pos = remove_bytes; + bb->flip(); + auto st = ctx->body_sink->append(bb); + // schema_buffer stores 1M of data for parsing column information + // need to determine whether to cache for the first time + if (ctx->is_read_schema) { + if (ctx->schema_buffer->pos + remove_bytes < config::stream_tvf_buffer_size) { + ctx->schema_buffer->put_bytes(bb->ptr, remove_bytes); + } else { + LOG(INFO) << "use a portion of data to request fe to obtain column information"; + ctx->is_read_schema = false; + ctx->status = process_put(req, ctx); + } } + if (!st.ok() && !ctx->status.ok()) { + LOG(WARNING) << "append body content failed. errmsg=" << st << ", " << ctx->brief(); + ctx->status = st; + return; + } + ctx->receive_bytes += remove_bytes; + } catch (const doris::Exception& e) { + if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { + ctx->status = Status::MemoryLimitExceeded( + fmt::format("PreCatch error code:{}, {}, ", e.code(), e.to_string())); + } + ctx->status = Status::Error(e.code(), e.to_string()); } - - if (!st.ok() && !ctx->status.ok()) { - LOG(WARNING) << "append body content failed. errmsg=" << st << ", " << ctx->brief(); - ctx->status = st; - return; - } - ctx->receive_bytes += remove_bytes; } // after all the data has been read and it has not reached 1M, it will execute here if (ctx->is_read_schema) { diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp index 3f32655cf14027..3f2e5e93be250d 100644 --- a/be/src/http/action/stream_load.cpp +++ b/be/src/http/action/stream_load.cpp @@ -339,19 +339,29 @@ void StreamLoadAction::on_chunk_data(HttpRequest* req) { struct evhttp_request* ev_req = req->get_evhttp_request(); auto evbuf = evhttp_request_get_input_buffer(ev_req); + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->stream_load_pipe_tracker()); + int64_t start_read_data_time = MonotonicNanos(); while (evbuffer_get_length(evbuf) > 0) { - auto bb = ByteBuffer::allocate(128 * 1024); - auto remove_bytes = evbuffer_remove(evbuf, bb->ptr, bb->capacity); - bb->pos = remove_bytes; - bb->flip(); - auto st = ctx->body_sink->append(bb); - if (!st.ok()) { - LOG(WARNING) << "append body content failed. errmsg=" << st << ", " << ctx->brief(); - ctx->status = st; - return; - } - ctx->receive_bytes += remove_bytes; + try { + auto bb = ByteBuffer::allocate(128 * 1024); + auto remove_bytes = evbuffer_remove(evbuf, bb->ptr, bb->capacity); + bb->pos = remove_bytes; + bb->flip(); + auto st = ctx->body_sink->append(bb); + if (!st.ok()) { + LOG(WARNING) << "append body content failed. errmsg=" << st << ", " << ctx->brief(); + ctx->status = st; + return; + } + ctx->receive_bytes += remove_bytes; + } catch (const doris::Exception& e) { + if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { + ctx->status = Status::MemoryLimitExceeded( + fmt::format("PreCatch error code:{}, {}, ", e.code(), e.to_string())); + } + ctx->status = Status::Error(e.code(), e.to_string()); + } } int64_t read_data_time = MonotonicNanos() - start_read_data_time; int64_t last_receive_and_read_data_cost_nanos = ctx->receive_and_read_data_cost_nanos; diff --git a/be/src/io/fs/stream_load_pipe.cpp b/be/src/io/fs/stream_load_pipe.cpp index ecce306bdf1ad2..b1b3055beb7ecb 100644 --- a/be/src/io/fs/stream_load_pipe.cpp +++ b/be/src/io/fs/stream_load_pipe.cpp @@ -111,7 +111,9 @@ Status StreamLoadPipe::read_one_message(std::unique_ptr* data, size_t } Status StreamLoadPipe::append_and_flush(const char* data, size_t size, size_t proto_byte_size) { - ByteBufferPtr buf = ByteBuffer::allocate(BitUtil::RoundUpToPowerOfTwo(size + 1)); + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->stream_load_pipe_tracker()); + ByteBufferPtr buf; + RETURN_IF_ERROR_OR_CATCH_EXCEPTION(ByteBuffer::create_and_allocate(buf, 128 * 1024)); buf->put_bytes(data, size); buf->flip(); return _append(buf, proto_byte_size); @@ -145,7 +147,8 @@ Status StreamLoadPipe::append(const char* data, size_t size) { // need to allocate a new chunk, min chunk is 64k size_t chunk_size = std::max(_min_chunk_size, size - pos); chunk_size = BitUtil::RoundUpToPowerOfTwo(chunk_size); - _write_buf = ByteBuffer::allocate(chunk_size); + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->stream_load_pipe_tracker()); + RETURN_IF_ERROR_OR_CATCH_EXCEPTION(ByteBuffer::create_and_allocate(_write_buf, chunk_size)); _write_buf->put_bytes(data + pos, size - pos); return Status::OK(); } diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 4b2478ccf99b4b..83929cd999feeb 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -185,6 +185,9 @@ class ExecEnv { std::shared_ptr segcompaction_mem_tracker() { return _segcompaction_mem_tracker; } + std::shared_ptr stream_load_pipe_tracker() { + return _stream_load_pipe_tracker; + } std::shared_ptr rowid_storage_reader_tracker() { return _rowid_storage_reader_tracker; } @@ -362,6 +365,7 @@ class ExecEnv { std::shared_ptr _brpc_iobuf_block_memory_tracker; // Count the memory consumption of segment compaction tasks. std::shared_ptr _segcompaction_mem_tracker; + std::shared_ptr _stream_load_pipe_tracker; // TODO, looking forward to more accurate tracking. std::shared_ptr _rowid_storage_reader_tracker; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index bbd6bbc9447fbf..bce801a7a3dd52 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -535,6 +535,8 @@ void ExecEnv::init_mem_tracker() { MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "SubcolumnsTree"); _s3_file_buffer_tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "S3FileBuffer"); + _stream_load_pipe_tracker = + MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::GLOBAL, "StreamLoadPipe"); } void ExecEnv::_register_metrics() { diff --git a/be/src/runtime/stream_load/stream_load_context.h b/be/src/runtime/stream_load/stream_load_context.h index 1dc7ccf73ba18b..60540d46a85d0c 100644 --- a/be/src/runtime/stream_load/stream_load_context.h +++ b/be/src/runtime/stream_load/stream_load_context.h @@ -37,6 +37,7 @@ #include "common/utils.h" #include "runtime/exec_env.h" #include "runtime/stream_load/stream_load_executor.h" +#include "runtime/thread_context.h" #include "util/byte_buffer.h" #include "util/time.h" #include "util/uid_util.h" @@ -95,9 +96,14 @@ class StreamLoadContext { public: StreamLoadContext(ExecEnv* exec_env) : id(UniqueId::gen_uid()), _exec_env(exec_env) { start_millis = UnixMillis(); + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->stream_load_pipe_tracker()); + schema_buffer = ByteBuffer::allocate(config::stream_tvf_buffer_size); } ~StreamLoadContext() { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER( + ExecEnv::GetInstance()->stream_load_pipe_tracker()); + schema_buffer.reset(); if (need_rollback) { _exec_env->stream_load_executor()->rollback_txn(this); need_rollback = false; @@ -182,7 +188,7 @@ class StreamLoadContext { std::shared_ptr body_sink; std::shared_ptr pipe; - ByteBufferPtr schema_buffer = ByteBuffer::allocate(config::stream_tvf_buffer_size); + ByteBufferPtr schema_buffer; TStreamLoadPutResult put_result; TStreamLoadMultiTablePutResult multi_table_put_result; diff --git a/be/src/runtime/stream_load/stream_load_executor.cpp b/be/src/runtime/stream_load/stream_load_executor.cpp index 0761b445bee084..0616c6474aad32 100644 --- a/be/src/runtime/stream_load/stream_load_executor.cpp +++ b/be/src/runtime/stream_load/stream_load_executor.cpp @@ -142,6 +142,10 @@ Status StreamLoadExecutor::execute_plan_fragment(std::shared_ptr; -struct ByteBuffer { +struct ByteBuffer : private Allocator { static ByteBufferPtr allocate(size_t size) { ByteBufferPtr ptr(new ByteBuffer(size)); return ptr; } - ~ByteBuffer() { delete[] ptr; } + static Status create_and_allocate(ByteBufferPtr& ptr, size_t size) { + ptr = ByteBufferPtr(new ByteBuffer(size)); + return Status::OK(); + } + + ~ByteBuffer() { Allocator::free(ptr, capacity); } void put_bytes(const char* data, size_t size) { memcpy(ptr + pos, data, size); @@ -56,14 +64,15 @@ struct ByteBuffer { size_t remaining() const { return limit - pos; } bool has_remaining() const { return limit > pos; } - char* const ptr; + char* ptr; size_t pos; size_t limit; size_t capacity; private: - ByteBuffer(size_t capacity_) - : ptr(new char[capacity_]), pos(0), limit(capacity_), capacity(capacity_) {} + ByteBuffer(size_t capacity_) : pos(0), limit(capacity_), capacity(capacity_) { + ptr = reinterpret_cast(Allocator::alloc(capacity_)); + } }; } // namespace doris From dc7a90ced512b004645049813a7cc2a68ed2389b Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Thu, 29 Aug 2024 20:20:32 +0800 Subject: [PATCH 6/7] [enhancement](exception) catch exception for streamload and validate column (#40092) --- be/src/http/action/http_stream.cpp | 58 ++++++++++--------- be/src/http/action/stream_load.cpp | 35 +++++------ be/src/io/file_factory.cpp | 3 +- be/src/io/fs/stream_load_pipe.cpp | 4 +- .../runtime/stream_load/stream_load_context.h | 11 ++++ be/src/util/byte_buffer.h | 9 +-- be/src/vec/sink/vtablet_block_convertor.cpp | 11 ++-- be/src/vec/sink/vtablet_block_convertor.h | 13 ++++- be/test/util/byte_buffer2_test.cpp | 3 +- 9 files changed, 82 insertions(+), 65 deletions(-) diff --git a/be/src/http/action/http_stream.cpp b/be/src/http/action/http_stream.cpp index cd9b10bea975c6..83ce0ce82cc638 100644 --- a/be/src/http/action/http_stream.cpp +++ b/be/src/http/action/http_stream.cpp @@ -237,37 +237,39 @@ void HttpStreamAction::on_chunk_data(HttpRequest* req) { SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->stream_load_pipe_tracker()); int64_t start_read_data_time = MonotonicNanos(); + Status st = ctx->allocate_schema_buffer(); + if (!st.ok()) { + ctx->status = st; + return; + } while (evbuffer_get_length(evbuf) > 0) { - try { - auto bb = ByteBuffer::allocate(128 * 1024); - auto remove_bytes = evbuffer_remove(evbuf, bb->ptr, bb->capacity); - bb->pos = remove_bytes; - bb->flip(); - auto st = ctx->body_sink->append(bb); - // schema_buffer stores 1M of data for parsing column information - // need to determine whether to cache for the first time - if (ctx->is_read_schema) { - if (ctx->schema_buffer->pos + remove_bytes < config::stream_tvf_buffer_size) { - ctx->schema_buffer->put_bytes(bb->ptr, remove_bytes); - } else { - LOG(INFO) << "use a portion of data to request fe to obtain column information"; - ctx->is_read_schema = false; - ctx->status = process_put(req, ctx); - } - } - if (!st.ok() && !ctx->status.ok()) { - LOG(WARNING) << "append body content failed. errmsg=" << st << ", " << ctx->brief(); - ctx->status = st; - return; - } - ctx->receive_bytes += remove_bytes; - } catch (const doris::Exception& e) { - if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { - ctx->status = Status::MemoryLimitExceeded( - fmt::format("PreCatch error code:{}, {}, ", e.code(), e.to_string())); + ByteBufferPtr bb; + st = ByteBuffer::allocate(128 * 1024, &bb); + if (!st.ok()) { + ctx->status = st; + return; + } + auto remove_bytes = evbuffer_remove(evbuf, bb->ptr, bb->capacity); + bb->pos = remove_bytes; + bb->flip(); + st = ctx->body_sink->append(bb); + // schema_buffer stores 1M of data for parsing column information + // need to determine whether to cache for the first time + if (ctx->is_read_schema) { + if (ctx->schema_buffer()->pos + remove_bytes < config::stream_tvf_buffer_size) { + ctx->schema_buffer()->put_bytes(bb->ptr, remove_bytes); + } else { + LOG(INFO) << "use a portion of data to request fe to obtain column information"; + ctx->is_read_schema = false; + ctx->status = process_put(req, ctx); } - ctx->status = Status::Error(e.code(), e.to_string()); } + if (!st.ok()) { + LOG(WARNING) << "append body content failed. errmsg=" << st << ", " << ctx->brief(); + ctx->status = st; + return; + } + ctx->receive_bytes += remove_bytes; } // after all the data has been read and it has not reached 1M, it will execute here if (ctx->is_read_schema) { diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp index 3f2e5e93be250d..2036043b4d40e4 100644 --- a/be/src/http/action/stream_load.cpp +++ b/be/src/http/action/stream_load.cpp @@ -343,25 +343,22 @@ void StreamLoadAction::on_chunk_data(HttpRequest* req) { int64_t start_read_data_time = MonotonicNanos(); while (evbuffer_get_length(evbuf) > 0) { - try { - auto bb = ByteBuffer::allocate(128 * 1024); - auto remove_bytes = evbuffer_remove(evbuf, bb->ptr, bb->capacity); - bb->pos = remove_bytes; - bb->flip(); - auto st = ctx->body_sink->append(bb); - if (!st.ok()) { - LOG(WARNING) << "append body content failed. errmsg=" << st << ", " << ctx->brief(); - ctx->status = st; - return; - } - ctx->receive_bytes += remove_bytes; - } catch (const doris::Exception& e) { - if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { - ctx->status = Status::MemoryLimitExceeded( - fmt::format("PreCatch error code:{}, {}, ", e.code(), e.to_string())); - } - ctx->status = Status::Error(e.code(), e.to_string()); - } + ByteBufferPtr bb; + Status st = ByteBuffer::allocate(128 * 1024, &bb); + if (!st.ok()) { + ctx->status = st; + return; + } + auto remove_bytes = evbuffer_remove(evbuf, bb->ptr, bb->capacity); + bb->pos = remove_bytes; + bb->flip(); + st = ctx->body_sink->append(bb); + if (!st.ok()) { + LOG(WARNING) << "append body content failed. errmsg=" << st << ", " << ctx->brief(); + ctx->status = st; + return; + } + ctx->receive_bytes += remove_bytes; } int64_t read_data_time = MonotonicNanos() - start_read_data_time; int64_t last_receive_and_read_data_cost_nanos = ctx->receive_and_read_data_cost_nanos; diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index 4d6158f8f7e96d..1d9ff11de957bc 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -161,7 +161,8 @@ Status FileFactory::create_pipe_reader(const TUniqueId& load_id, io::FileReaderS if (!stream_load_ctx) { return Status::InternalError("unknown stream load id: {}", UniqueId(load_id).to_string()); } - if (need_schema == true) { + if (need_schema) { + RETURN_IF_ERROR(stream_load_ctx->allocate_schema_buffer()); // Here, a portion of the data is processed to parse column information auto pipe = std::make_shared( io::kMaxPipeBufferedBytes /* max_buffered_bytes */, 64 * 1024 /* min_chunk_size */, diff --git a/be/src/io/fs/stream_load_pipe.cpp b/be/src/io/fs/stream_load_pipe.cpp index b1b3055beb7ecb..392125e6fc0b86 100644 --- a/be/src/io/fs/stream_load_pipe.cpp +++ b/be/src/io/fs/stream_load_pipe.cpp @@ -113,7 +113,7 @@ Status StreamLoadPipe::read_one_message(std::unique_ptr* data, size_t Status StreamLoadPipe::append_and_flush(const char* data, size_t size, size_t proto_byte_size) { SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->stream_load_pipe_tracker()); ByteBufferPtr buf; - RETURN_IF_ERROR_OR_CATCH_EXCEPTION(ByteBuffer::create_and_allocate(buf, 128 * 1024)); + RETURN_IF_ERROR(ByteBuffer::allocate(128 * 1024, &buf)); buf->put_bytes(data, size); buf->flip(); return _append(buf, proto_byte_size); @@ -148,7 +148,7 @@ Status StreamLoadPipe::append(const char* data, size_t size) { size_t chunk_size = std::max(_min_chunk_size, size - pos); chunk_size = BitUtil::RoundUpToPowerOfTwo(chunk_size); SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->stream_load_pipe_tracker()); - RETURN_IF_ERROR_OR_CATCH_EXCEPTION(ByteBuffer::create_and_allocate(_write_buf, chunk_size)); + RETURN_IF_ERROR(ByteBuffer::allocate(chunk_size, &_write_buf)); _write_buf->put_bytes(data + pos, size - pos); return Status::OK(); } diff --git a/be/src/runtime/stream_load/stream_load_context.h b/be/src/runtime/stream_load/stream_load_context.h index 60540d46a85d0c..098db5514c103f 100644 --- a/be/src/runtime/stream_load/stream_load_context.h +++ b/be/src/runtime/stream_load/stream_load_context.h @@ -124,6 +124,17 @@ class StreamLoadContext { // also print the load source info if detail is set to true std::string brief(bool detail = false) const; + Status allocate_schema_buffer() { + if (_schema_buffer == nullptr) { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER( + ExecEnv::GetInstance()->stream_load_pipe_tracker()); + return ByteBuffer::allocate(config::stream_tvf_buffer_size, &_schema_buffer); + } + return Status::OK(); + } + + ByteBufferPtr schema_buffer() { return _schema_buffer; } + public: static const int default_txn_id = -1; // load type, eg: ROUTINE LOAD/MANUAL LOAD diff --git a/be/src/util/byte_buffer.h b/be/src/util/byte_buffer.h index e8eadf69e028b5..6bb19dd2a88f2d 100644 --- a/be/src/util/byte_buffer.h +++ b/be/src/util/byte_buffer.h @@ -33,13 +33,8 @@ struct ByteBuffer; using ByteBufferPtr = std::shared_ptr; struct ByteBuffer : private Allocator { - static ByteBufferPtr allocate(size_t size) { - ByteBufferPtr ptr(new ByteBuffer(size)); - return ptr; - } - - static Status create_and_allocate(ByteBufferPtr& ptr, size_t size) { - ptr = ByteBufferPtr(new ByteBuffer(size)); + static Status allocate(const size_t size, ByteBufferPtr* ptr) { + RETURN_IF_CATCH_EXCEPTION({ *ptr = ByteBufferPtr(new ByteBuffer(size)); }); return Status::OK(); } diff --git a/be/src/vec/sink/vtablet_block_convertor.cpp b/be/src/vec/sink/vtablet_block_convertor.cpp index 96de68f597677c..086c9a3ddd04d9 100644 --- a/be/src/vec/sink/vtablet_block_convertor.cpp +++ b/be/src/vec/sink/vtablet_block_convertor.cpp @@ -182,12 +182,11 @@ DecimalType OlapTableBlockConvertor::_get_decimalv3_min_or_max(const TypeDescrip return DecimalType(value); } -Status OlapTableBlockConvertor::_validate_column(RuntimeState* state, const TypeDescriptor& type, - bool is_nullable, vectorized::ColumnPtr column, - size_t slot_index, bool* stop_processing, - fmt::memory_buffer& error_prefix, - const uint32_t row_count, - vectorized::IColumn::Permutation* rows) { +Status OlapTableBlockConvertor::_internal_validate_column( + RuntimeState* state, const TypeDescriptor& type, bool is_nullable, + vectorized::ColumnPtr column, size_t slot_index, bool* stop_processing, + fmt::memory_buffer& error_prefix, const uint32_t row_count, + vectorized::IColumn::Permutation* rows) { DCHECK((rows == nullptr) || (rows->size() == row_count)); fmt::memory_buffer error_msg; auto set_invalid_and_append_error_msg = [&](int row) { diff --git a/be/src/vec/sink/vtablet_block_convertor.h b/be/src/vec/sink/vtablet_block_convertor.h index 0db340ce6c27d4..7f866c38032775 100644 --- a/be/src/vec/sink/vtablet_block_convertor.h +++ b/be/src/vec/sink/vtablet_block_convertor.h @@ -69,7 +69,18 @@ class OlapTableBlockConvertor { Status _validate_column(RuntimeState* state, const TypeDescriptor& type, bool is_nullable, vectorized::ColumnPtr column, size_t slot_index, bool* stop_processing, fmt::memory_buffer& error_prefix, const uint32_t row_count, - vectorized::IColumn::Permutation* rows = nullptr); + vectorized::IColumn::Permutation* rows = nullptr) { + RETURN_IF_CATCH_EXCEPTION({ + return _internal_validate_column(state, type, is_nullable, column, slot_index, + stop_processing, error_prefix, row_count, rows); + }); + } + + Status _internal_validate_column(RuntimeState* state, const TypeDescriptor& type, + bool is_nullable, vectorized::ColumnPtr column, + size_t slot_index, bool* stop_processing, + fmt::memory_buffer& error_prefix, const uint32_t row_count, + vectorized::IColumn::Permutation* rows = nullptr); // make input data valid for OLAP table // return number of invalid/filtered rows. diff --git a/be/test/util/byte_buffer2_test.cpp b/be/test/util/byte_buffer2_test.cpp index 04b62cd5fe8f0e..73c38c9e404340 100644 --- a/be/test/util/byte_buffer2_test.cpp +++ b/be/test/util/byte_buffer2_test.cpp @@ -32,7 +32,8 @@ class ByteBufferTest : public testing::Test { }; TEST_F(ByteBufferTest, normal) { - auto buf = ByteBuffer::allocate(4); + ByteBufferPtr buf; + Status st = ByteBuffer::allocate(4, &buf); EXPECT_EQ(0, buf->pos); EXPECT_EQ(4, buf->limit); EXPECT_EQ(4, buf->capacity); From 485f2b7314d5fe88a119b3d126a188288f9fec9a Mon Sep 17 00:00:00 2001 From: yiguolei Date: Mon, 9 Sep 2024 14:22:09 +0800 Subject: [PATCH 7/7] f --- be/src/io/file_factory.cpp | 6 ++--- be/src/runtime/runtime_state.cpp | 26 ------------------- .../runtime/stream_load/stream_load_context.h | 8 +----- be/src/vec/common/schema_util.cpp | 7 +++-- 4 files changed, 7 insertions(+), 40 deletions(-) diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index 1d9ff11de957bc..95d537320883e8 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -166,9 +166,9 @@ Status FileFactory::create_pipe_reader(const TUniqueId& load_id, io::FileReaderS // Here, a portion of the data is processed to parse column information auto pipe = std::make_shared( io::kMaxPipeBufferedBytes /* max_buffered_bytes */, 64 * 1024 /* min_chunk_size */, - stream_load_ctx->schema_buffer->pos /* total_length */); - stream_load_ctx->schema_buffer->flip(); - RETURN_IF_ERROR(pipe->append(stream_load_ctx->schema_buffer)); + stream_load_ctx->schema_buffer()->pos /* total_length */); + stream_load_ctx->schema_buffer()->flip(); + RETURN_IF_ERROR(pipe->append(stream_load_ctx->schema_buffer())); RETURN_IF_ERROR(pipe->finish()); *file_reader = std::move(pipe); } else { diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 82c1198e9332ea..cdb5a65a977147 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -463,32 +463,6 @@ Status RuntimeState::append_error_msg_to_file(std::function line, return Status::OK(); } -std::string RuntimeState::get_error_log_file_path() { - if (_s3_error_fs && _error_log_file && _error_log_file->is_open()) { - // close error log file - _error_log_file->close(); - std::string error_log_absolute_path = - _exec_env->load_path_mgr()->get_load_error_absolute_path(_error_log_file_path); - // upload error log file to s3 - Status st = _s3_error_fs->upload(error_log_absolute_path, _s3_error_log_file_path); - if (st.ok()) { - // remove local error log file - std::filesystem::remove(error_log_absolute_path); - } else { - // upload failed and return local error log file path - LOG(WARNING) << "Fail to upload error file to s3, error_log_file_path=" - << _error_log_file_path << ", error=" << st; - return _error_log_file_path; - } - // expiration must be less than a week (in seconds) for presigned url - static const unsigned EXPIRATION_SECONDS = 7 * 24 * 60 * 60 - 1; - // We should return a public endpoint to user. - _error_log_file_path = _s3_error_fs->generate_presigned_url(_s3_error_log_file_path, - EXPIRATION_SECONDS, true); - } - return _error_log_file_path; -} - void RuntimeState::resize_op_id_to_local_state(int operator_size) { _op_id_to_local_state.resize(-operator_size); } diff --git a/be/src/runtime/stream_load/stream_load_context.h b/be/src/runtime/stream_load/stream_load_context.h index 098db5514c103f..2ccf8ce5014a88 100644 --- a/be/src/runtime/stream_load/stream_load_context.h +++ b/be/src/runtime/stream_load/stream_load_context.h @@ -96,14 +96,9 @@ class StreamLoadContext { public: StreamLoadContext(ExecEnv* exec_env) : id(UniqueId::gen_uid()), _exec_env(exec_env) { start_millis = UnixMillis(); - SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->stream_load_pipe_tracker()); - schema_buffer = ByteBuffer::allocate(config::stream_tvf_buffer_size); } ~StreamLoadContext() { - SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER( - ExecEnv::GetInstance()->stream_load_pipe_tracker()); - schema_buffer.reset(); if (need_rollback) { _exec_env->stream_load_executor()->rollback_txn(this); need_rollback = false; @@ -199,8 +194,6 @@ class StreamLoadContext { std::shared_ptr body_sink; std::shared_ptr pipe; - ByteBufferPtr schema_buffer; - TStreamLoadPutResult put_result; TStreamLoadMultiTablePutResult multi_table_put_result; @@ -258,6 +251,7 @@ class StreamLoadContext { private: ExecEnv* _exec_env = nullptr; + ByteBufferPtr _schema_buffer; }; } // namespace doris diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index 4db526ed9b5ba4..61b98bafd48ef6 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -544,11 +544,10 @@ Status _parse_variant_columns(Block& block, const std::vector& variant_pos, } Status parse_variant_columns(Block& block, const std::vector& variant_pos, - const ParseConfig& config) { + const ParseContext& ctx) { // Parse each variant column from raw string column - RETURN_IF_CATCH_EXCEPTION({ - return vectorized::schema_util::_parse_variant_columns(block, variant_pos, config); - }); + RETURN_IF_CATCH_EXCEPTION( + { return vectorized::schema_util::_parse_variant_columns(block, variant_pos, ctx); }); } void finalize_variant_columns(Block& block, const std::vector& variant_pos,