Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[opt](memory) Process profile add metadata profile #44647

Merged
merged 1 commit into from
Nov 27, 2024
Merged

[opt](memory) Process profile add metadata profile #44647

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 135 additions & 5 deletions be/src/olap/metadata_adder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,16 @@
#include <bvar/bvar.h>
#include <stdint.h>

#include "util/runtime_profile.h"

namespace doris {

inline bvar::Adder<int64_t> g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");

inline bvar::Adder<int64_t> g_all_rowsets_mem_bytes("doris_all_rowsets_mem_bytes");
inline bvar::Adder<int64_t> g_all_rowsets_num("doris_all_rowsets_num");

inline bvar::Adder<int64_t> g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");

Expand All @@ -37,8 +42,8 @@ inline bvar::Adder<int64_t> g_tablet_index_num("doris_tablet_index_num");
inline bvar::Adder<int64_t> g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");

inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
inline bvar::Adder<int64_t> g_all_segments_mem_bytes("doris_all_segments_mem_bytes");
inline bvar::Adder<int64_t> g_all_segments_num("doris_all_segments_num");

inline bvar::Adder<int64_t> g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
Expand Down Expand Up @@ -104,6 +109,8 @@ class MetadataAdder {
public:
MetadataAdder();

static void dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot);

protected:
MetadataAdder(const MetadataAdder& other);

Expand Down Expand Up @@ -159,6 +166,8 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
}
if constexpr (std::is_same_v<T, RowsetMeta>) {
g_rowset_meta_mem_bytes << val;
} else if constexpr (std::is_same_v<T, Rowset>) {
g_all_rowsets_mem_bytes << val;
} else if constexpr (std::is_same_v<T, TabletMeta>) {
g_tablet_meta_mem_bytes << val;
} else if constexpr (std::is_same_v<T, TabletColumn>) {
Expand All @@ -168,7 +177,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
} else if constexpr (std::is_same_v<T, TabletSchema>) {
g_tablet_schema_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
g_segment_mem_bytes << val;
g_all_segments_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
g_column_reader_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
Expand All @@ -185,6 +194,9 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
g_ordinal_index_reader_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
g_zone_map_index_reader_mem_bytes << val;
} else {
LOG(FATAL) << "add_mem_size not match class type: " << typeid(T).name() << ", " << val;
__builtin_unreachable();
}
}

Expand All @@ -195,6 +207,8 @@ void MetadataAdder<T>::add_num(int64_t val) {
}
if constexpr (std::is_same_v<T, RowsetMeta>) {
g_rowset_meta_num << val;
} else if constexpr (std::is_same_v<T, Rowset>) {
g_all_rowsets_num << val;
} else if constexpr (std::is_same_v<T, TabletMeta>) {
g_tablet_meta_num << val;
} else if constexpr (std::is_same_v<T, TabletColumn>) {
Expand All @@ -204,7 +218,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
} else if constexpr (std::is_same_v<T, TabletSchema>) {
g_tablet_schema_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
g_segment_num << val;
g_all_segments_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
g_column_reader_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
Expand All @@ -221,7 +235,123 @@ void MetadataAdder<T>::add_num(int64_t val) {
g_ordinal_index_reader_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
g_zone_map_index_reader_num << val;
} else {
LOG(FATAL) << "add_num not match class type: " << typeid(T).name() << ", " << val;
__builtin_unreachable();
}
}

}; // namespace doris
template <typename T>
void MetadataAdder<T>::dump_metadata_object(RuntimeProfile* object_heap_dump_snapshot) {
RuntimeProfile::Counter* rowset_meta_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "RowsetMetaMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* rowset_meta_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "RowsetMetaNum", TUnit::UNIT);
COUNTER_SET(rowset_meta_mem_bytes_counter, g_rowset_meta_mem_bytes.get_value());
COUNTER_SET(rowset_meta_num_counter, g_rowset_meta_num.get_value());

RuntimeProfile::Counter* all_rowsets_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* all_rowsets_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "AllRowsetsNum", TUnit::UNIT);
COUNTER_SET(all_rowsets_mem_bytes_counter, g_all_rowsets_mem_bytes.get_value());
COUNTER_SET(all_rowsets_num_counter, g_all_rowsets_num.get_value());

RuntimeProfile::Counter* tablet_meta_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* tablet_meta_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletMetaNum", TUnit::UNIT);
COUNTER_SET(tablet_meta_mem_bytes_counter, g_tablet_meta_mem_bytes.get_value());
COUNTER_SET(tablet_meta_num_counter, g_tablet_meta_num.get_value());

RuntimeProfile::Counter* tablet_column_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletColumnMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* tablet_column_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletColumnNum", TUnit::UNIT);
COUNTER_SET(tablet_column_mem_bytes_counter, g_tablet_column_mem_bytes.get_value());
COUNTER_SET(tablet_column_num_counter, g_tablet_column_num.get_value());

RuntimeProfile::Counter* tablet_index_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletIndexMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* tablet_index_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletIndexNum", TUnit::UNIT);
COUNTER_SET(tablet_index_mem_bytes_counter, g_tablet_index_mem_bytes.get_value());
COUNTER_SET(tablet_index_num_counter, g_tablet_index_num.get_value());

RuntimeProfile::Counter* tablet_schema_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletSchemaMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* tablet_schema_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "TabletSchemaNum", TUnit::UNIT);
COUNTER_SET(tablet_schema_mem_bytes_counter, g_tablet_schema_mem_bytes.get_value());
COUNTER_SET(tablet_schema_num_counter, g_tablet_schema_num.get_value());

RuntimeProfile::Counter* all_segments_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* all_segments_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "AllSegmentsNum", TUnit::UNIT);
COUNTER_SET(all_segments_mem_bytes_counter, g_all_segments_mem_bytes.get_value());
COUNTER_SET(all_segments_num_counter, g_all_segments_num.get_value());

RuntimeProfile::Counter* column_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* column_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "ColumnReaderNum", TUnit::UNIT);
COUNTER_SET(column_reader_mem_bytes_counter, g_column_reader_mem_bytes.get_value());
COUNTER_SET(column_reader_num_counter, g_column_reader_num.get_value());

RuntimeProfile::Counter* bitmap_index_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "BitmapIndexReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* bitmap_index_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "BitmapIndexReaderNum", TUnit::UNIT);
COUNTER_SET(bitmap_index_reader_mem_bytes_counter, g_bitmap_index_reader_mem_bytes.get_value());
COUNTER_SET(bitmap_index_reader_num_counter, g_bitmap_index_reader_num.get_value());

RuntimeProfile::Counter* bloom_filter_index_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "BloomFilterIndexReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* filter_index_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "BloomFilterIndexReaderNum", TUnit::UNIT);
COUNTER_SET(bloom_filter_index_reader_mem_bytes_counter,
g_bloom_filter_index_reader_mem_bytes.get_value());
COUNTER_SET(filter_index_reader_num_counter, g_bloom_filter_index_reader_num.get_value());

RuntimeProfile::Counter* index_page_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "IndexPageReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* index_page_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "IndexPageReaderNum", TUnit::UNIT);
COUNTER_SET(index_page_reader_mem_bytes_counter, g_index_page_reader_mem_bytes.get_value());
COUNTER_SET(index_page_reader_num_counter, g_index_page_reader_num.get_value());

RuntimeProfile::Counter* indexed_column_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "IndexedColumnReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* indexed_column_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "IndexedColumnReaderNum", TUnit::UNIT);
COUNTER_SET(indexed_column_reader_mem_bytes_counter,
g_indexed_column_reader_mem_bytes.get_value());
COUNTER_SET(indexed_column_reader_num_counter, g_indexed_column_reader_num.get_value());

RuntimeProfile::Counter* inverted_index_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "InvertedIndexReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* inverted_index_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "InvertedIndexReaderNum", TUnit::UNIT);
COUNTER_SET(inverted_index_reader_mem_bytes_counter,
g_inverted_index_reader_mem_bytes.get_value());
COUNTER_SET(inverted_index_reader_num_counter, g_inverted_index_reader_num.get_value());

RuntimeProfile::Counter* ordinal_index_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "OrdinalIndexReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* ordinal_index_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "OrdinalIndexReaderNum", TUnit::UNIT);
COUNTER_SET(ordinal_index_reader_mem_bytes_counter,
g_ordinal_index_reader_mem_bytes.get_value());
COUNTER_SET(ordinal_index_reader_num_counter, g_ordinal_index_reader_num.get_value());

RuntimeProfile::Counter* zone_map_index_reader_mem_bytes_counter =
ADD_COUNTER(object_heap_dump_snapshot, "ZoneMapIndexReaderMemBytes", TUnit::BYTES);
RuntimeProfile::Counter* zone_map_index_reader_num_counter =
ADD_COUNTER(object_heap_dump_snapshot, "ZoneMapIndexReaderNum", TUnit::UNIT);
COUNTER_SET(zone_map_index_reader_mem_bytes_counter,
g_zone_map_index_reader_mem_bytes.get_value());
COUNTER_SET(zone_map_index_reader_num_counter, g_zone_map_index_reader_num.get_value());
}

}; // namespace doris
7 changes: 0 additions & 7 deletions be/src/olap/rowset/rowset.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@

namespace doris {

static bvar::Adder<size_t> g_total_rowset_num("doris_total_rowset_num");

Rowset::Rowset(const TabletSchemaSPtr& schema, RowsetMetaSharedPtr rowset_meta,
std::string tablet_path)
: _rowset_meta(std::move(rowset_meta)),
Expand Down Expand Up @@ -56,11 +54,6 @@ Rowset::Rowset(const TabletSchemaSPtr& schema, RowsetMetaSharedPtr rowset_meta,
}
// build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema
_schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() : schema;
g_total_rowset_num << 1;
}

Rowset::~Rowset() {
g_total_rowset_num << -1;
}

Status Rowset::load(bool use_cache) {
Expand Down
5 changes: 2 additions & 3 deletions be/src/olap/rowset/rowset.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

#include "common/logging.h"
#include "common/status.h"
#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/rowset_meta.h"
#include "olap/tablet_schema.h"
Expand Down Expand Up @@ -116,10 +117,8 @@ class RowsetStateMachine {
RowsetState _rowset_state;
};

class Rowset : public std::enable_shared_from_this<Rowset> {
class Rowset : public std::enable_shared_from_this<Rowset>, public MetadataAdder<Rowset> {
public:
virtual ~Rowset();

// Open all segment files in this rowset and load necessary metadata.
// - `use_cache` : whether to use fd cache, only applicable to alpha rowset now
//
Expand Down
10 changes: 9 additions & 1 deletion be/src/runtime/process_profile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <memory>

#include "olap/metadata_adder.h"
#include "runtime/memory/memory_profile.h"

namespace doris {
Expand All @@ -37,8 +38,15 @@ void ProcessProfile::refresh_profile() {
std::unique_ptr<RuntimeProfile> process_profile =
std::make_unique<RuntimeProfile>("ProcessProfile");
_memory_profile->make_memory_profile(process_profile.get());
_process_profile.set(std::move(process_profile));
// TODO make other profile

// 3. dump object heap
RuntimeProfile* object_heap_dump_snapshot =
process_profile->create_child("ObjectHeapDump", true, false);
MetadataAdder<ProcessProfile>::dump_metadata_object(object_heap_dump_snapshot);
// TODO dump other object (block, column, etc.)

_process_profile.set(std::move(process_profile));
}

} // namespace doris
Loading