Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[chore] Add bvar for meta operations of BE #28374

Merged
merged 4 commits into from
Dec 19, 2023
Merged

[chore] Add bvar for meta operations of BE #28374

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
31b5889
[chore] Add bvar for meta operations of BE
gavinchou Dec 13, 2023
415264f
Use bvar latency recorder instead
gavinchou Dec 14, 2023
b08223e
Fix format
gavinchou Dec 14, 2023
49900c0
Remove latency in name
gavinchou Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 12 additions & 23 deletions be/src/olap/olap_meta.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include "olap/olap_meta.h"

#include <bvar/latency_recorder.h>
#include <fmt/format.h>
#include <fmt/ranges.h>
#include <rocksdb/env.h>
Expand Down Expand Up @@ -56,6 +57,10 @@ using namespace ErrorCode;
const std::string META_POSTFIX = "/meta";
const size_t PREFIX_LENGTH = 4;

bvar::LatencyRecorder g_meta_put_latency("meta_put");
bvar::LatencyRecorder g_meta_get_latency("meta_get");
bvar::LatencyRecorder g_meta_remove_latency("meta_remove");

OlapMeta::OlapMeta(const std::string& root_path) : _root_path(root_path) {}

OlapMeta::~OlapMeta() = default;
Expand Down Expand Up @@ -117,15 +122,14 @@ Status OlapMeta::init() {
}

Status OlapMeta::get(const int column_family_index, const std::string& key, std::string* value) {
DorisMetrics::instance()->meta_read_request_total->increment(1);
auto& handle = _handles[column_family_index];
int64_t duration_ns = 0;
rocksdb::Status s;
{
SCOPED_RAW_TIMER(&duration_ns);
s = _db->Get(ReadOptions(), handle.get(), rocksdb::Slice(key), value);
}
DorisMetrics::instance()->meta_read_request_duration_us->increment(duration_ns / 1000);
g_meta_get_latency << (duration_ns / 1000);
if (s.IsNotFound()) {
return Status::Error<META_KEY_NOT_FOUND>("OlapMeta::get meet not found key");
} else if (!s.ok()) {
Expand All @@ -137,23 +141,20 @@ Status OlapMeta::get(const int column_family_index, const std::string& key, std:

bool OlapMeta::key_may_exist(const int column_family_index, const std::string& key,
std::string* value) {
DorisMetrics::instance()->meta_read_request_total->increment(1);
auto& handle = _handles[column_family_index];
int64_t duration_ns = 0;
bool is_exist = false;
{
SCOPED_RAW_TIMER(&duration_ns);
is_exist = _db->KeyMayExist(ReadOptions(), handle.get(), rocksdb::Slice(key), value);
}
DorisMetrics::instance()->meta_read_request_duration_us->increment(duration_ns / 1000);
g_meta_get_latency << (duration_ns / 1000);

return is_exist;
}

Status OlapMeta::put(const int column_family_index, const std::string& key,
const std::string& value) {
DorisMetrics::instance()->meta_write_request_total->increment(1);

// log all params
VLOG_DEBUG << "column_family_index: " << column_family_index << ", key: " << key
<< ", value: " << value;
Expand All @@ -162,9 +163,7 @@ Status OlapMeta::put(const int column_family_index, const std::string& key,
rocksdb::Status s;
{
int64_t duration_ns = 0;
Defer defer([&] {
DorisMetrics::instance()->meta_write_request_duration_us->increment(duration_ns / 1000);
});
Defer defer([&] { g_meta_put_latency << (duration_ns / 1000); });
SCOPED_RAW_TIMER(&duration_ns);

WriteOptions write_options;
Expand All @@ -180,15 +179,11 @@ Status OlapMeta::put(const int column_family_index, const std::string& key,
}

Status OlapMeta::put(const int column_family_index, const std::vector<BatchEntry>& entries) {
DorisMetrics::instance()->meta_write_request_total->increment(1);

auto* handle = _handles[column_family_index].get();
rocksdb::Status s;
{
int64_t duration_ns = 0;
Defer defer([&] {
DorisMetrics::instance()->meta_write_request_duration_us->increment(duration_ns / 1000);
});
Defer defer([&] { g_meta_put_latency << (duration_ns / 1000); });
SCOPED_RAW_TIMER(&duration_ns);

// construct write batch
Expand All @@ -212,14 +207,10 @@ Status OlapMeta::put(const int column_family_index, const std::vector<BatchEntry
}

Status OlapMeta::put(rocksdb::WriteBatch* batch) {
DorisMetrics::instance()->meta_write_request_total->increment(1);

rocksdb::Status s;
{
int64_t duration_ns = 0;
Defer defer([&] {
DorisMetrics::instance()->meta_write_request_duration_us->increment(duration_ns / 1000);
});
Defer defer([&] { g_meta_put_latency << (duration_ns / 1000); });
SCOPED_RAW_TIMER(&duration_ns);

WriteOptions write_options;
Expand All @@ -234,7 +225,6 @@ Status OlapMeta::put(rocksdb::WriteBatch* batch) {
}

Status OlapMeta::remove(const int column_family_index, const std::string& key) {
DorisMetrics::instance()->meta_write_request_total->increment(1);
auto& handle = _handles[column_family_index];
rocksdb::Status s;
int64_t duration_ns = 0;
Expand All @@ -244,7 +234,7 @@ Status OlapMeta::remove(const int column_family_index, const std::string& key) {
write_options.sync = config::sync_tablet_meta;
s = _db->Delete(write_options, handle.get(), rocksdb::Slice(key));
}
DorisMetrics::instance()->meta_write_request_duration_us->increment(duration_ns / 1000);
g_meta_remove_latency << (duration_ns / 1000);
if (!s.ok()) {
return Status::Error<META_DELETE_ERROR>("rocks db delete key: {}, failed, reason: {}", key,
s.ToString());
Expand All @@ -253,7 +243,6 @@ Status OlapMeta::remove(const int column_family_index, const std::string& key) {
}

Status OlapMeta::remove(const int column_family_index, const std::vector<std::string>& keys) {
DorisMetrics::instance()->meta_write_request_total->increment(1);
auto& handle = _handles[column_family_index];
rocksdb::Status s;
int64_t duration_ns = 0;
Expand All @@ -267,7 +256,7 @@ Status OlapMeta::remove(const int column_family_index, const std::vector<std::st
}
s = _db->Write(write_options, &batch);
}
DorisMetrics::instance()->meta_write_request_duration_us->increment(duration_ns / 1000);
g_meta_remove_latency << (duration_ns / 1000);
if (!s.ok()) {
return Status::Error<META_DELETE_ERROR>("rocks db delete keys:{} failed, reason:{}", keys,
s.ToString());
Expand Down
15 changes: 0 additions & 15 deletions be/src/util/doris_metrics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,16 +92,6 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(base_compaction_bytes_total, MetricUnit::BY
DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(cumulative_compaction_bytes_total, MetricUnit::BYTES, "",
compaction_bytes_total, Labels({{"type", "cumulative"}}));

DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(meta_write_request_total, MetricUnit::REQUESTS, "",
meta_request_total, Labels({{"type", "write"}}));
DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(meta_read_request_total, MetricUnit::REQUESTS, "",
meta_request_total, Labels({{"type", "read"}}));

DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(meta_write_request_duration_us, MetricUnit::MICROSECONDS, "",
meta_request_duration, Labels({{"type", "write"}}));
DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(meta_read_request_duration_us, MetricUnit::MICROSECONDS, "",
meta_request_duration, Labels({{"type", "read"}}));

DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(segment_read_total, MetricUnit::OPERATIONS,
"(segment_v2) total number of segments read", segment_read,
Labels({{"type", "segment_read_total"}}));
Expand Down Expand Up @@ -237,11 +227,6 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) {
INT_COUNTER_METRIC_REGISTER(_server_metric_entity, cumulative_compaction_deltas_total);
INT_COUNTER_METRIC_REGISTER(_server_metric_entity, cumulative_compaction_bytes_total);

INT_COUNTER_METRIC_REGISTER(_server_metric_entity, meta_write_request_total);
INT_COUNTER_METRIC_REGISTER(_server_metric_entity, meta_write_request_duration_us);
INT_COUNTER_METRIC_REGISTER(_server_metric_entity, meta_read_request_total);
INT_COUNTER_METRIC_REGISTER(_server_metric_entity, meta_read_request_duration_us);

INT_COUNTER_METRIC_REGISTER(_server_metric_entity, segment_read_total);
INT_COUNTER_METRIC_REGISTER(_server_metric_entity, segment_row_total);

Expand Down
5 changes: 0 additions & 5 deletions be/src/util/doris_metrics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,6 @@ class DorisMetrics {
IntCounter* publish_task_request_total = nullptr;
IntCounter* publish_task_failed_total = nullptr;

IntCounter* meta_write_request_total = nullptr;
IntCounter* meta_write_request_duration_us = nullptr;
IntCounter* meta_read_request_total = nullptr;
IntCounter* meta_read_request_duration_us = nullptr;

// Counters for segment_v2
// -----------------------
// total number of segments read
Expand Down
Loading