diff --git a/be/src/common/config.h b/be/src/common/config.h index c9a87909ffd974..d70f9353357019 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -598,6 +598,12 @@ CONF_Int32(aws_log_level, "3"); // the buffer size when read data from remote storage like s3 CONF_mInt32(remote_storage_read_buffer_mb, "256"); +// Default level of MemTracker to show in web page +// now MemTracker support two level: +// RELEASE: 0 +// DEBUG: 1 +// the level equal or lower than mem_tracker_level will show in web page +CONF_Int16(mem_tracker_level, "0"); } // namespace config } // namespace doris diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index 76273d726b83d0..10ce02d5b4645c 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -47,7 +47,11 @@ OlapScanner::OlapScanner(RuntimeState* runtime_state, OlapScanNode* parent, bool _aggregation(aggregation), _need_agg_finalize(need_agg_finalize), _tuple_idx(parent->_tuple_idx), - _direct_conjunct_size(parent->_direct_conjunct_size) { + _direct_conjunct_size(parent->_direct_conjunct_size), + _mem_tracker(MemTracker::CreateTracker(runtime_state->fragment_mem_tracker()->limit(), + "OlapScanner", + runtime_state->fragment_mem_tracker(), + true, true, MemTrackerLevel::DEBUG)) { _reader.reset(new Reader()); DCHECK(_reader.get() != NULL); @@ -90,7 +94,7 @@ Status OlapScanner::prepare(const TPaloScanRange& scan_range, // the rowsets maybe compacted when the last olap scanner starts Version rd_version(0, _version); OLAPStatus acquire_reader_st = - _tablet->capture_rs_readers(rd_version, &_params.rs_readers); + _tablet->capture_rs_readers(rd_version, &_params.rs_readers, _mem_tracker); if (acquire_reader_st != OLAP_SUCCESS) { LOG(WARNING) << "fail to init reader.res=" << acquire_reader_st; std::stringstream ss; @@ -248,11 +252,7 @@ Status OlapScanner::get_batch(RuntimeState* state, RowBatch* batch, bool* eof) { bzero(tuple_buf, state->batch_size() * _tuple_desc->byte_size()); Tuple* tuple = reinterpret_cast(tuple_buf); - auto tracker = MemTracker::CreateTracker(state->fragment_mem_tracker()->limit(), - "OlapScanner:" + print_id(state->query_id()), - state->fragment_mem_tracker()); - std::unique_ptr mem_pool(new MemPool(tracker.get())); - + std::unique_ptr mem_pool(new MemPool(_mem_tracker.get())); int64_t raw_rows_threshold = raw_rows_read() + config::doris_scanner_row_num; { SCOPED_TIMER(_parent->_scan_timer); diff --git a/be/src/exec/olap_scanner.h b/be/src/exec/olap_scanner.h index 08b6035acbea49..15cf13becab4d7 100644 --- a/be/src/exec/olap_scanner.h +++ b/be/src/exec/olap_scanner.h @@ -145,6 +145,8 @@ class OlapScanner { bool _is_closed = false; MonotonicStopWatch _watcher; + + std::shared_ptr _mem_tracker; }; } // namespace doris diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp index 5208941ce00976..60b1706a8244c7 100644 --- a/be/src/exec/tablet_sink.cpp +++ b/be/src/exec/tablet_sink.cpp @@ -534,7 +534,7 @@ Status OlapTableSink::prepare(RuntimeState* state) { // profile must add to state's object pool _profile = state->obj_pool()->add(new RuntimeProfile("OlapTableSink")); _mem_tracker = MemTracker::CreateTracker(-1, "OlapTableSink:" + std::to_string(state->load_job_id()), - state->instance_mem_tracker()); + state->instance_mem_tracker(), true, false); SCOPED_TIMER(_profile->total_time_counter()); diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 6a47c89da9a969..ee57e3990f78f4 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -28,9 +28,11 @@ namespace doris { Compaction::Compaction(TabletSharedPtr tablet, const std::string& label, const std::shared_ptr& parent_tracker) - : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker)), - _readers_tracker(MemTracker::CreateTracker(-1, "CompactionReaderTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker)), - _writer_tracker(MemTracker::CreateTracker(-1, "CompationWriterTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker)), + : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker, true, false)), + _readers_tracker(MemTracker::CreateTracker(-1, "CompactionReaderTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker, + true, false)), + _writer_tracker(MemTracker::CreateTracker(-1, "CompationWriterTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker, + true, false)), _tablet(tablet), _input_rowsets_size(0), _input_row_num(0), @@ -171,7 +173,7 @@ OLAPStatus Compaction::construct_input_rowset_readers() { RETURN_NOT_OK(rowset->create_reader( MemTracker::CreateTracker( -1, "Compaction:RowsetReader:" + rowset->rowset_id().to_string(), - _readers_tracker), + _readers_tracker, true, true, MemTrackerLevel::DEBUG), &rs_reader)); _input_rs_readers.push_back(std::move(rs_reader)); } diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp index 81da1b84340264..0abf4286a7fc56 100644 --- a/be/src/olap/lru_cache.cpp +++ b/be/src/olap/lru_cache.cpp @@ -400,7 +400,7 @@ uint32_t ShardedLRUCache::_shard(uint32_t hash) { ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t total_capacity, std::shared_ptr parent) : _name(name), _last_id(1), - _mem_tracker(MemTracker::CreateTracker(-1, name, parent, true)) { + _mem_tracker(MemTracker::CreateTracker(-1, name, parent, true, false)) { const size_t per_shard = (total_capacity + (kNumShards - 1)) / kNumShards; for (int s = 0; s < kNumShards; s++) { _shards[s].set_capacity(per_shard); diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 4ee8b4ad51cab8..f728af55faaa50 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -40,7 +40,7 @@ MemTable::MemTable(int64_t tablet_id, Schema* schema, const TabletSchema* tablet _slot_descs(slot_descs), _keys_type(keys_type), _row_comparator(_schema), - _mem_tracker(MemTracker::CreateTracker(-1, "MemTable:" + std::to_string(tablet_id), parent_tracker)), + _mem_tracker(MemTracker::CreateTracker(-1, "MemTable", parent_tracker)), _buffer_mem_pool(new MemPool(_mem_tracker.get())), _table_mem_pool(new MemPool(_mem_tracker.get())), _schema_size(_schema->schema_size()), diff --git a/be/src/olap/row_block2.cpp b/be/src/olap/row_block2.cpp index 2b940bd16d54e6..94d600d3007e91 100644 --- a/be/src/olap/row_block2.cpp +++ b/be/src/olap/row_block2.cpp @@ -18,6 +18,7 @@ #include "olap/row_block2.h" #include +#include #include "gutil/strings/substitute.h" #include "olap/row_cursor.h" @@ -33,7 +34,7 @@ RowBlockV2::RowBlockV2(const Schema& schema, uint16_t capacity, std::shared_ptr< : _schema(schema), _capacity(capacity), _column_vector_batches(_schema.num_columns()), - _tracker(MemTracker::CreateTracker(-1, "RowBlockV2", parent)), + _tracker(MemTracker::CreateTracker(-1, "RowBlockV2", std::move(parent))), _pool(new MemPool(_tracker.get())), _selection_vector(nullptr) { for (auto cid : _schema.column_ids()) { diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index 0f5899cb67c9fc..7a3c49bdf3fc59 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -17,6 +17,8 @@ #include "beta_rowset_reader.h" +#include + #include "olap/delete_handler.h" #include "olap/generic_iterators.h" #include "olap/row_block.h" @@ -28,12 +30,17 @@ namespace doris { BetaRowsetReader::BetaRowsetReader(BetaRowsetSharedPtr rowset, - const std::shared_ptr& parent_tracker) - : _rowset(std::move(rowset)), _stats(&_owned_stats), _parent_tracker(parent_tracker) { + std::shared_ptr parent_tracker) + : _rowset(std::move(rowset)), _stats(&_owned_stats), _parent_tracker(std::move(parent_tracker)) { _rowset->aquire(); } OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) { + // If do not init the RowsetReader with a parent_tracker, use the runtime_state instance_mem_tracker + if (_parent_tracker == nullptr && read_context->runtime_state != nullptr) { + _parent_tracker = read_context->runtime_state->instance_mem_tracker(); + } + RETURN_NOT_OK(_rowset->load(true, _parent_tracker)); _context = read_context; if (_context->stats != nullptr) { @@ -110,13 +117,9 @@ OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) { // init input block _input_block.reset(new RowBlockV2(schema, 1024, _parent_tracker)); - // init output block and row - if (_parent_tracker == nullptr && read_context->runtime_state != nullptr) { - _output_block.reset(new RowBlock(read_context->tablet_schema, - read_context->runtime_state->instance_mem_tracker())); - } else { - _output_block.reset(new RowBlock(read_context->tablet_schema, _parent_tracker)); - } + // init input/output block and row + _output_block.reset(new RowBlock(read_context->tablet_schema, _parent_tracker)); + RowBlockInfo output_block_info; output_block_info.row_num = 1024; output_block_info.null_supported = true; diff --git a/be/src/olap/rowset/beta_rowset_reader.h b/be/src/olap/rowset/beta_rowset_reader.h index 1318d89044c5c9..6d8a3f1bede833 100644 --- a/be/src/olap/rowset/beta_rowset_reader.h +++ b/be/src/olap/rowset/beta_rowset_reader.h @@ -30,7 +30,7 @@ namespace doris { class BetaRowsetReader : public RowsetReader { public: BetaRowsetReader(BetaRowsetSharedPtr rowset, - const std::shared_ptr& parent_tracker = nullptr); + std::shared_ptr parent_tracker = nullptr); ~BetaRowsetReader() override { _rowset->release(); } diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index f8e6024b1196f7..992d8c414d30e8 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -46,7 +46,7 @@ Status Segment::open(std::string filename, uint32_t segment_id, const TabletSche Segment::Segment(std::string fname, uint32_t segment_id, const TabletSchema* tablet_schema, std::shared_ptr parent) : _fname(std::move(fname)), _segment_id(segment_id), - _tablet_schema(tablet_schema), _mem_tracker(MemTracker::CreateTracker(-1, "Segment", std::move(parent), false)) {} + _tablet_schema(tablet_schema), _mem_tracker(MemTracker::CreateTracker(-1, "Segment", std::move(parent), false, true)) {} Segment::~Segment() { _mem_tracker->Release(_mem_tracker->consumption()); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 67c5adf7c135e6..859cca50c49157 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -189,7 +189,7 @@ Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_ra } } _seek_schema.reset(new Schema(key_fields, key_fields.size())); - _seek_block.reset(new RowBlockV2(*_seek_schema, 1)); + _seek_block.reset(new RowBlockV2(*_seek_schema, 1, _mem_tracker)); // create used column iterator for (auto cid : _seek_schema->column_ids()) { diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index e7388977c19b4c..7d69f604a9aa5d 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1338,7 +1338,7 @@ bool SchemaChangeWithSorting::_external_sorting(vector& src_row std::vector rs_readers; for (auto& rowset : src_rowsets) { RowsetReaderSharedPtr rs_reader; - auto res = rowset->create_reader(&rs_reader); + auto res = rowset->create_reader(_mem_tracker, &rs_reader); if (res != OLAP_SUCCESS) { LOG(WARNING) << "failed to create rowset reader."; return false; @@ -1473,6 +1473,9 @@ OLAPStatus SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe // for schema change, seek_columns is the same to return_columns reader_context.seek_columns = &return_columns; + auto mem_tracker = MemTracker::CreateTracker(-1, "AlterTablet:" + std::to_string(base_tablet->tablet_id()) + "-" + + std::to_string(new_tablet->tablet_id()), _mem_tracker, true, false, MemTrackerLevel::DEBUG); + do { // get history data to be converted and it will check if there is hold in base tablet res = _get_versions_to_be_changed(base_tablet, &versions_to_be_changed); @@ -1535,7 +1538,7 @@ OLAPStatus SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe } // acquire data sources correspond to history versions - base_tablet->capture_rs_readers(versions_to_be_changed, &rs_readers); + base_tablet->capture_rs_readers(versions_to_be_changed, &rs_readers, mem_tracker); if (rs_readers.size() < 1) { LOG(WARNING) << "fail to acquire all data sources. " << "version_num=" << versions_to_be_changed.size() @@ -1688,7 +1691,7 @@ OLAPStatus SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tabl reader_context.seek_columns = &return_columns; RowsetReaderSharedPtr rowset_reader; - RETURN_NOT_OK((*base_rowset)->create_reader(&rowset_reader)); + RETURN_NOT_OK((*base_rowset)->create_reader(_mem_tracker, &rowset_reader)); rowset_reader->init(&reader_context); RowsetWriterContext writer_context; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 7866f0f69ba081..7921d04eec1961 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -628,15 +628,17 @@ OLAPStatus Tablet::_capture_consistent_rowsets_unlocked( } OLAPStatus Tablet::capture_rs_readers(const Version& spec_version, - std::vector* rs_readers) const { + std::vector* rs_readers, + std::shared_ptr parent_tracker) const { std::vector version_path; RETURN_NOT_OK(capture_consistent_versions(spec_version, &version_path)); - RETURN_NOT_OK(capture_rs_readers(version_path, rs_readers)); + RETURN_NOT_OK(capture_rs_readers(version_path, rs_readers, parent_tracker)); return OLAP_SUCCESS; } OLAPStatus Tablet::capture_rs_readers(const std::vector& version_path, - std::vector* rs_readers) const { + std::vector* rs_readers, + std::shared_ptr parent_tracker) const { DCHECK(rs_readers != nullptr && rs_readers->empty()); for (auto version : version_path) { auto it = _rs_version_map.find(version); @@ -653,7 +655,7 @@ OLAPStatus Tablet::capture_rs_readers(const std::vector& version_path, } } RowsetReaderSharedPtr rs_reader; - auto res = it->second->create_reader(&rs_reader); + auto res = it->second->create_reader(parent_tracker, &rs_reader); if (res != OLAP_SUCCESS) { LOG(WARNING) << "failed to create reader for rowset:" << it->second->rowset_id(); return OLAP_ERR_CAPTURE_ROWSET_READER_ERROR; diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 1102fb54168144..7da632e44c8c16 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -121,9 +121,12 @@ class Tablet : public BaseTablet { OLAPStatus capture_consistent_rowsets(const Version& spec_version, vector* rowsets) const; OLAPStatus capture_rs_readers(const Version& spec_version, - vector* rs_readers) const; + vector* rs_readers, + std::shared_ptr parent_tracker = nullptr) const; + OLAPStatus capture_rs_readers(const vector& version_path, - vector* rs_readers) const; + vector* rs_readers, + std::shared_ptr parent_tracker = nullptr) const; DelPredicateArray delete_predicates() { return _tablet_meta->delete_predicates(); } void add_delete_predicate(const DeletePredicatePB& delete_predicate, int64_t version); diff --git a/be/src/runtime/data_stream_recvr.cc b/be/src/runtime/data_stream_recvr.cc index fd3a536a307f9b..cba524e152ca54 100644 --- a/be/src/runtime/data_stream_recvr.cc +++ b/be/src/runtime/data_stream_recvr.cc @@ -447,7 +447,7 @@ DataStreamRecvr::DataStreamRecvr( _profile(profile), _sub_plan_query_statistics_recvr(sub_plan_query_statistics_recvr) { _mem_tracker = MemTracker::CreateTracker( - _profile, -1, "DataStreamRecvr:" + print_id(_fragment_instance_id), parent_tracker); + _profile, -1, "DataStreamRecvr", parent_tracker); // Create one queue per sender if is_merging is true. int num_queues = is_merging ? num_senders : 1; diff --git a/be/src/runtime/data_stream_sender.cpp b/be/src/runtime/data_stream_sender.cpp index 343baf50433297..a8a500b10bb967 100644 --- a/be/src/runtime/data_stream_sender.cpp +++ b/be/src/runtime/data_stream_sender.cpp @@ -469,8 +469,7 @@ Status DataStreamSender::prepare(RuntimeState* state) { _profile = _pool->add(new RuntimeProfile(title.str())); SCOPED_TIMER(_profile->total_time_counter()); _mem_tracker = MemTracker::CreateTracker( - _profile, -1, "DataStreamSender:" + print_id(state->fragment_instance_id()), - state->instance_mem_tracker()); + _profile, -1, "DataStreamSender", state->instance_mem_tracker()); if (_part_type == TPartitionType::UNPARTITIONED || _part_type == TPartitionType::RANDOM) { // Randomize the order we open/transmit to channels to avoid thundering herd problems. diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 39b9a2a5fcdf5d..7bd31a32e47c26 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -185,8 +185,8 @@ Status ExecEnv::_init_mem_tracker() { return Status::InternalError(ss.str()); } - _mem_tracker = - MemTracker::CreateTracker(bytes_limit, "Query", MemTracker::GetRootTracker()); + _mem_tracker = MemTracker::CreateTracker(bytes_limit, "Query", MemTracker::GetRootTracker(), + false, false); LOG(INFO) << "Using global memory limit: " << PrettyPrinter::print(bytes_limit, TUnit::BYTES); RETURN_IF_ERROR(_disk_io_mgr->init(_mem_tracker)); diff --git a/be/src/runtime/export_sink.cpp b/be/src/runtime/export_sink.cpp index 0e88bb4987e8df..5e4d2cc9855296 100644 --- a/be/src/runtime/export_sink.cpp +++ b/be/src/runtime/export_sink.cpp @@ -72,8 +72,7 @@ Status ExportSink::prepare(RuntimeState* state) { _mem_tracker = MemTracker::CreateTracker( -1, - "ExportSink:" + print_id(state->fragment_instance_id()), - state->instance_mem_tracker()); + "ExportSink", state->instance_mem_tracker()); // Prepare the exprs to run. RETURN_IF_ERROR(Expr::prepare(_output_expr_ctxs, state, _row_desc, _mem_tracker)); diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp index 30c2f2f4e77881..d6ade09e89bfa8 100644 --- a/be/src/runtime/load_channel.cpp +++ b/be/src/runtime/load_channel.cpp @@ -27,7 +27,7 @@ LoadChannel::LoadChannel(const UniqueId& load_id, int64_t mem_limit, int64_t tim const std::shared_ptr& mem_tracker) : _load_id(load_id), _timeout_s(timeout_s) { _mem_tracker = MemTracker::CreateTracker( - mem_limit, "LoadChannel:" + _load_id.to_string(), mem_tracker); + mem_limit, "LoadChannel:" + _load_id.to_string(), mem_tracker, true, false); // _last_updated_time should be set before being inserted to // _load_channels in load_channel_mgr, or it may be erased // immediately by gc thread. diff --git a/be/src/runtime/mem_tracker.cpp b/be/src/runtime/mem_tracker.cpp index 2c5fac45ad9f55..bdb803054c4db1 100644 --- a/be/src/runtime/mem_tracker.cpp +++ b/be/src/runtime/mem_tracker.cpp @@ -17,9 +17,11 @@ #include "runtime/mem_tracker.h" -#include +#include #include +#include + #include #include @@ -75,15 +77,32 @@ void MemTracker::CreateRootTracker() { } std::shared_ptr MemTracker::CreateTracker(RuntimeProfile* profile, int64_t byte_limit, - const std::string& label, - const std::shared_ptr& parent) { - shared_ptr real_parent; + const std::string& label, const std::shared_ptr& parent, + bool reset_label_name, MemTrackerLevel level) { + std::shared_ptr real_parent; + std::string label_name; + // if parent is not null, reset label name to query id. + // The parent label always: RuntimeState:instance:8ca5a59e3aa84f74-84bb0d0466193736 + // we just need the last id of it: 8ca5a59e3aa84f74-84bb0d0466193736 + // to build the new label name of tracker: `label`: 8ca5a59e3aa84f74-84bb0d0466193736 + // else if parent is null + // just use the root is parent and keep the label_name as label if (parent) { - real_parent = std::move(parent); + real_parent = parent; + if (reset_label_name) { + std::vector tmp_result; + boost::split(tmp_result, parent->label(), boost::is_any_of(":")); + label_name = label + ":" + tmp_result[tmp_result.size() - 1]; + } else { + label_name = label; + } } else { real_parent = GetRootTracker(); + label_name = label; } - shared_ptr tracker(new MemTracker(profile, byte_limit, label, real_parent, true)); + + shared_ptr tracker(new MemTracker(profile, byte_limit, label_name, real_parent, true, + level > real_parent->_level ? level : real_parent->_level)); real_parent->AddChildTracker(tracker); tracker->Init(); @@ -91,16 +110,32 @@ std::shared_ptr MemTracker::CreateTracker(RuntimeProfile* profile, i } std::shared_ptr MemTracker::CreateTracker(int64_t byte_limit, const std::string& label, - std::shared_ptr parent, - bool log_usage_if_zero) { - shared_ptr real_parent; + std::shared_ptr parent, bool log_usage_if_zero, bool reset_label_name, MemTrackerLevel level) { + std::shared_ptr real_parent; + std::string label_name; + // if parent is not null, reset label name to query id. + // The parent label always: RuntimeState:instance:8ca5a59e3aa84f74-84bb0d0466193736 + // we just need the last id of it: 8ca5a59e3aa84f74-84bb0d0466193736 + // to build the new label name of tracker: `label`: 8ca5a59e3aa84f74-84bb0d0466193736 + // else if parent is null + // just use the root is parent and keep the label_name as label if (parent) { - real_parent = std::move(parent); + real_parent = parent; + if (reset_label_name) { + std::vector tmp_result; + boost::split(tmp_result, parent->label(), boost::is_any_of(":")); + label_name = label + ":" + tmp_result[tmp_result.size() - 1]; + } else { + label_name = label; + } } else { real_parent = GetRootTracker(); + label_name = label; } + shared_ptr tracker( - new MemTracker(nullptr, byte_limit, label, real_parent, log_usage_if_zero)); + new MemTracker(nullptr, byte_limit, label_name, real_parent, log_usage_if_zero, + level > real_parent->_level ? level : real_parent->_level)); real_parent->AddChildTracker(tracker); tracker->Init(); @@ -108,16 +143,17 @@ std::shared_ptr MemTracker::CreateTracker(int64_t byte_limit, const } MemTracker::MemTracker(int64_t byte_limit, const std::string& label) - : MemTracker(nullptr, byte_limit, label, std::shared_ptr(), true) {} + : MemTracker(nullptr, byte_limit, label, std::shared_ptr(), true, MemTrackerLevel::RELEASE) {} MemTracker::MemTracker(RuntimeProfile* profile, int64_t byte_limit, const string& label, - const std::shared_ptr& parent, bool log_usage_if_zero) + const std::shared_ptr& parent, bool log_usage_if_zero, MemTrackerLevel level) : limit_(byte_limit), soft_limit_(CalcSoftLimit(byte_limit)), label_(label), parent_(parent), consumption_metric_(nullptr), log_usage_if_zero_(log_usage_if_zero), + _level(level), num_gcs_metric_(nullptr), bytes_freed_by_last_gc_metric_(nullptr), bytes_over_limit_metric_(nullptr), @@ -259,7 +295,7 @@ void MemTracker::ListTrackers(vector>* trackers) { } for (const auto& child_weak : children) { shared_ptr child = child_weak.lock(); - if (child) { + if (child && static_cast(child->_level) <= config::mem_tracker_level) { to_process.emplace_back(std::move(child)); } } diff --git a/be/src/runtime/mem_tracker.h b/be/src/runtime/mem_tracker.h index 94f40fa0b07b76..142e6112999258 100644 --- a/be/src/runtime/mem_tracker.h +++ b/be/src/runtime/mem_tracker.h @@ -40,6 +40,10 @@ namespace doris { /// limit should be used. enum class MemLimit { HARD, SOFT }; +/// The Level use to decide whether to show it in web page +/// each MemTracker have a Level equals to parent, only be set explicit +enum class MemTrackerLevel {RELEASE = 0, DEBUG}; + class ObjectPool; class MemTracker; struct ReservationTrackerCounters; @@ -88,11 +92,11 @@ class MemTracker : public std::enable_shared_from_this { static std::shared_ptr CreateTracker( int64_t byte_limit = -1, const std::string& label = std::string(), std::shared_ptr parent = std::shared_ptr(), - bool log_usage_if_zero = true); + bool log_usage_if_zero = true, bool reset_label_name = true, MemTrackerLevel level = MemTrackerLevel::RELEASE); static std::shared_ptr CreateTracker( RuntimeProfile* profile, int64_t byte_limit, const std::string& label = std::string(), - const std::shared_ptr& parent = std::shared_ptr()); + const std::shared_ptr& parent = std::shared_ptr(), bool reset_label_name = true, MemTrackerLevel level = MemTrackerLevel::RELEASE); // this is used for creating an orphan mem tracker, or for unit test. // If a mem tracker has parent, it should be created by `CreateTracker()` @@ -420,7 +424,7 @@ class MemTracker : public std::enable_shared_from_this { /// If 'log_usage_if_zero' is false, this tracker (and its children) will not be /// included in LogUsage() output if consumption is 0. MemTracker(RuntimeProfile* profile, int64_t byte_limit, const std::string& label, - const std::shared_ptr& parent, bool log_usage_if_zero); + const std::shared_ptr& parent, bool log_usage_if_zero, MemTrackerLevel); private: friend class PoolMemTrackerRegistry; @@ -553,6 +557,8 @@ class MemTracker : public std::enable_shared_from_this { /// if consumption is 0. bool log_usage_if_zero_; + MemTrackerLevel _level; + /// The number of times the GcFunctions were called. IntCounter* num_gcs_metric_; diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp index 815d0d1fba12c0..3bd8a1be726f36 100644 --- a/be/src/runtime/plan_fragment_executor.cpp +++ b/be/src/runtime/plan_fragment_executor.cpp @@ -132,7 +132,7 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request, _mem_tracker = MemTracker::CreateTracker(bytes_limit, "PlanFragmentExecutor:" + print_id(_query_id) + ":" + print_id(params.fragment_instance_id), - _exec_env->process_mem_tracker()); + _exec_env->process_mem_tracker(), true, false); _runtime_state->set_fragment_mem_tracker(_mem_tracker); LOG(INFO) << "Using query memory limit: " << PrettyPrinter::print(bytes_limit, TUnit::BYTES); diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index b8c0c261e237d3..52376463c37f21 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -215,9 +215,9 @@ Status RuntimeState::init_mem_trackers(const TUniqueId& query_id) { _query_mem_tracker = MemTracker::CreateTracker(bytes_limit, "RuntimeState:query:" + print_id(query_id), - _exec_env->process_mem_tracker()); + _exec_env->process_mem_tracker(), true, false); _instance_mem_tracker = MemTracker::CreateTracker( - &_profile, -1, "RuntimeState:instance:" + print_id(query_id), _query_mem_tracker); + &_profile, -1, "RuntimeState:instance:", _query_mem_tracker); /* // TODO: this is a stopgap until we implement ExprContext diff --git a/be/src/runtime/tablets_channel.cpp b/be/src/runtime/tablets_channel.cpp index 5e647bb5c827ce..531a3bd9c9176d 100644 --- a/be/src/runtime/tablets_channel.cpp +++ b/be/src/runtime/tablets_channel.cpp @@ -34,7 +34,7 @@ std::atomic TabletsChannel::_s_tablet_writer_count; TabletsChannel::TabletsChannel(const TabletsChannelKey& key, const std::shared_ptr& mem_tracker) : _key(key), _state(kInitialized), _closed_senders(64) { - _mem_tracker = MemTracker::CreateTracker(-1, "TabletsChannel:" + key.to_string(), mem_tracker); + _mem_tracker = MemTracker::CreateTracker(-1, "TabletsChannel", mem_tracker); static std::once_flag once_flag; std::call_once(once_flag, [] { REGISTER_HOOK_METRIC(tablet_writer_count, [&]() { return _s_tablet_writer_count.load(); }); diff --git a/docs/en/administrator-guide/config/be_config.md b/docs/en/administrator-guide/config/be_config.md index 8dd31c19d64352..38ea221f69fac3 100644 --- a/docs/en/administrator-guide/config/be_config.md +++ b/docs/en/administrator-guide/config/be_config.md @@ -874,6 +874,7 @@ If the parameter is `THREAD_POOL`, the model is a blocking I/O model. ### `aws_log_level` * Type: int32 + * Description: log level of AWS SDK, ``` Off = 0, @@ -884,4 +885,15 @@ If the parameter is `THREAD_POOL`, the model is a blocking I/O model. Debug = 5, Trace = 6 ``` -* Default: 3 \ No newline at end of file + +* Default: 3 + +### `mem_tracker_level` + +* Type: int16 +* Description: The level at which MemTracker is displayed on the Web page equal or lower than this level will be displayed on the Web page + ``` + RELEASE = 0 + DEBUG = 1 + ``` +* Default: 0 \ No newline at end of file diff --git a/docs/zh-CN/administrator-guide/config/be_config.md b/docs/zh-CN/administrator-guide/config/be_config.md index 8f9756751f5c91..2b1f6ff1af4953 100644 --- a/docs/zh-CN/administrator-guide/config/be_config.md +++ b/docs/zh-CN/administrator-guide/config/be_config.md @@ -886,3 +886,13 @@ Stream Load 一般适用于导入几个GB以内的数据，不适合导入过大 ``` * 默认值: 3 + +### `mem_tracker_level` + +* 类型: int16 +* 描述: MemTracker在Web页面上展示的级别，等于或低于这个级别的MemTracker会在Web页面上展示 + ``` + RELEASE = 0 + DEBUG = 1 + ``` +* 默认值: 0