Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Perf Level By Bit Field #230

Closed
wants to merge 9 commits into from
Closed

Perf Level By Bit Field #230

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
9216d1c
implement perf level by bit field and redirect perf_level and its met…
LemonHX Mar 15, 2021
5e888e7
removing chinese comment
LemonHX Mar 15, 2021
02880fc
reformat code and rename the PerfLevelByBitField structure to PerfFlags.
LemonHX Mar 18, 2021
f1dae11
Reformat code and remove the usage of GetPerfLevel
LemonHX Mar 24, 2021
b5485ab
The data structure has been adjusted to make FFI more friendly. At th…
LemonHX Mar 26, 2021
c28cf61
Add FFI and restore the previous test cases and restore previous modi…
LemonHX Mar 29, 2021
a7357c3
git failed to diff this file for strage reason
LemonHX Mar 29, 2021
d39adb2
rollback design
LemonHX Apr 14, 2021
df2844c
removed nested if
LemonHX Apr 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -584,6 +584,7 @@ set(SOURCES
monitoring/instrumented_mutex.cc
monitoring/iostats_context.cc
monitoring/perf_context.cc
monitoring/perf_level_by_bitfield.cc
monitoring/perf_level.cc
monitoring/persistent_stats_history.cc
monitoring/statistics.cc
Expand Down
103 changes: 103 additions & 0 deletions include/rocksdb/perf_level_by_bitfield.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
#pragma once
#include <stdint.h>

namespace rocksdb {

struct PerfLevelByBitField {
LemonHX marked this conversation as resolved.
Show resolved Hide resolved
uint8_t perf_level;
LemonHX marked this conversation as resolved.
Show resolved Hide resolved

uint8_t enable_user_key_comparison_count_bit : 1; // 2
LemonHX marked this conversation as resolved.
Show resolved Hide resolved
uint8_t enable_block_cache_hit_count_bit : 1; // 2
uint8_t enable_block_read_count_bit : 1; // 2
LemonHX marked this conversation as resolved.
Show resolved Hide resolved
uint8_t enable_block_read_byte_bit : 1; // 2
uint8_t enable_block_cache_index_hit_count_bit : 1; // 2
uint8_t enable_index_block_read_count_bit : 1; // 2
uint8_t enable_block_cache_filter_hit_count_bit : 1; // 2
uint8_t enable_filter_block_read_count_bit : 1; // 2
uint8_t enable_compression_dict_block_read_count_bit : 1; // 2
uint8_t enable_get_read_bytes_bit : 1; // 2
uint8_t enable_multiget_read_bytes_bit : 1; // 2
uint8_t enable_iter_read_bytes_bit : 1; // 2
uint8_t enable_internal_key_skipped_count_bit : 1; // 2
uint8_t enable_internal_delete_skipped_count_bit : 1; // 2
uint8_t enable_internal_recent_skipped_count_bit : 1; // 2
uint8_t enable_internal_merge_count_bit : 1; // 2
uint8_t enable_get_from_memtable_count_bit : 1; // 2
uint8_t enable_seek_on_memtable_count_bit : 1; // 2
uint8_t enable_next_on_memtable_count_bit : 1; // 2
uint8_t enable_prev_on_memtable_count_bit : 1; // 2
uint8_t enable_seek_child_seek_count_bit : 1; // 2
uint8_t enable_bloom_memtable_hit_count_bit : 1; // 2
uint8_t enable_bloom_memtable_miss_count_bit : 1; // 2
uint8_t enable_bloom_sst_hit_count_bit : 1; // 2
uint8_t enable_bloom_sst_miss_count_bit : 1; // 2
uint8_t enable_key_lock_wait_count_bit : 1; // 2

// flag enable for using cpu time
uint8_t enable_measure_cpu_time_bit : 1; // 3 -> with CPU time flag
LemonHX marked this conversation as resolved.
Show resolved Hide resolved

uint8_t enable_block_read_time_bit : 1; // 3
uint8_t enable_block_checksum_time_bit : 1; // 3
uint8_t enable_block_decompress_time_bit : 1; // 3
uint8_t enable_get_snapshot_time_bit : 1; // 3
uint8_t enable_get_from_memtable_time_bit : 1; // 3
uint8_t enable_get_post_process_time_bit : 1; // 3
uint8_t enable_get_from_output_files_time_bit : 1; // 3
uint8_t enable_seek_on_memtable_time_bit : 1; // 3
uint8_t enable_seek_child_seek_time_bit : 1; // 3
uint8_t enable_seek_min_heap_time_bit : 1; // 3
uint8_t enable_seek_max_heap_time_bit : 1; // 3
uint8_t enable_seek_internal_seek_time_bit : 1; // 3
uint8_t enable_find_next_user_entry_time_bit : 1; // 3
uint8_t enable_write_wal_time_bit : 1; // 3
uint8_t enable_write_memtable_time_bit : 1; // 3
uint8_t enable_write_delay_time_bit : 1; // 3
uint8_t enable_write_scheduling_flushes_compactions_time_bit : 1; // 3
uint8_t enable_write_pre_and_post_process_time_bit : 1; // 3
uint8_t enable_write_thread_wait_nanos_bit : 1; // 3
uint8_t enable_merge_operator_time_nanos_bit : 1; // 3
uint8_t enable_read_index_block_nanos_bit : 1; // 3
uint8_t enable_read_filter_block_nanos_bit : 1; // 3
uint8_t enable_new_table_block_iter_nanos_bit : 1; // 3
uint8_t enable_new_table_iterator_nanos_bit : 1; // 3
uint8_t enable_block_seek_nanos_bit : 1; // 3
uint8_t enable_find_table_nanos_bit : 1; // 3
uint8_t enable_key_lock_wait_time_bit : 1; // 3
uint8_t enable_env_new_sequential_file_nanos_bit : 1; // 3
uint8_t enable_env_new_random_access_file_nanos_bit : 1; // 3
uint8_t enable_env_new_writable_file_nanos_bit : 1; // 3
uint8_t enable_env_reuse_writable_file_nanos_bit : 1; // 3
uint8_t enable_env_new_random_rw_file_nanos_bit : 1; // 3
uint8_t enable_env_new_directory_nanos_bit : 1; // 3
uint8_t enable_env_file_exists_nanos_bit : 1; // 3
uint8_t enable_env_get_children_nanos_bit : 1; // 3
uint8_t enable_env_get_children_file_attributes_nanos_bit : 1; // 3
uint8_t enable_env_delete_file_nanos_bit : 1; // 3
uint8_t enable_env_create_dir_nanos_bit : 1; // 3
uint8_t enable_env_create_dir_if_missing_nanos_bit : 1; // 3
uint8_t enable_env_delete_dir_nanos_bit : 1; // 3
uint8_t enable_env_get_file_size_nanos_bit : 1; // 3
uint8_t enable_env_get_file_modification_time_nanos_bit : 1; // 3
uint8_t enable_env_rename_file_nanos_bit : 1; // 3
uint8_t enable_env_link_file_nanos_bit : 1; // 3
uint8_t enable_env_lock_file_nanos_bit : 1; // 3
uint8_t enable_env_unlock_file_nanos_bit : 1; // 3
uint8_t enable_env_new_logger_nanos_bit : 1; // 3
uint8_t enable_encrypt_data_nanos_bit : 1; // 3
uint8_t enable_decrypt_data_nanos_bit : 1; // 3

uint8_t enable_get_cpu_nanos_bit : 1; // 4
uint8_t enable_iter_next_cpu_nanos_bit : 1; // 4
uint8_t enable_iter_prev_cpu_nanos_bit : 1; // 4
uint8_t enable_iter_seek_cpu_nanos_bit : 1; // 4

uint8_t enable_db_mutex_lock_nanos_bit : 1; // 5
uint8_t enable_db_condition_wait_nanos_bit : 1; // 5
};

// set the perf stats bitfield for current thread
void SetPerfBitField(PerfLevelByBitField pbf);

// get current perf stats bitfield for current thread
PerfLevelByBitField* GetPerfBitField();
} // namespace rocksdb
LemonHX marked this conversation as resolved.
Show resolved Hide resolved
82 changes: 46 additions & 36 deletions monitoring/perf_context_imp.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
// (found in the LICENSE.Apache file in the root directory).
//
#pragma once
#include "monitoring/perf_level_by_bitfield_imp.h"
#include "monitoring/perf_step_timer.h"
#include "rocksdb/perf_context.h"
#include "util/stop_watch.h"
Expand All @@ -22,12 +23,16 @@ extern thread_local PerfContext perf_context;

#if defined(NPERF_CONTEXT)

#define PERF_TIMER_GUARD(metric)
#define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition)
#define PERF_TIMER_MEASURE(metric)
#define PERF_TIMER_STOP(metric)
#define PERF_TIMER_START(metric)
#define PERF_TIMER_GUARD(metric)
#define PERF_TIMER_GUARD_WITH_ENV(metric, env)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's interesting that the original one does not have this definition.. Is it a mistake in RocksDB?

#define PERF_CPU_TIMER_GUARD(metric, env)
#define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition, stats, \
ticker_type)
#define PERF_TIMER_MEASURE(metric)
#define PERF_COUNTER_ADD(metric, value)
#define PERF_COUNTER_BY_LEVEL_ADD(metric, value, level)

#else

Expand All @@ -37,57 +42,62 @@ extern thread_local PerfContext perf_context;
#define PERF_TIMER_START(metric) perf_step_timer_##metric.Start();

// Declare and set start time of the timer
#define PERF_TIMER_GUARD(metric) \
PerfStepTimer perf_step_timer_##metric(&(perf_context.metric)); \
#define PERF_TIMER_GUARD(metric) \
PerfStepTimer perf_step_timer_##metric( \
&(perf_context.metric), nullptr, \
(bool)perf_bit_field.enable_measure_cpu_time_bit, \
(bool)perf_bit_field.enable_##metric##_bit); \
perf_step_timer_##metric.Start();

// Declare and set start time of the timer
#define PERF_TIMER_GUARD_WITH_ENV(metric, env) \
PerfStepTimer perf_step_timer_##metric(&(perf_context.metric), env); \
#define PERF_TIMER_GUARD_WITH_ENV(metric, env) \
PerfStepTimer perf_step_timer_##metric( \
&(perf_context.metric), env, \
(bool)perf_bit_field.enable_measure_cpu_time_bit, \
(bool)perf_bit_field.enable_##metric##_bit); \
perf_step_timer_##metric.Start();

// Declare and set start time of the timer
#define PERF_CPU_TIMER_GUARD(metric, env) \
PerfStepTimer perf_step_timer_##metric( \
&(perf_context.metric), env, true, \
PerfLevel::kEnableTimeAndCPUTimeExceptForMutex); \
#define PERF_CPU_TIMER_GUARD(metric, env) \
PerfStepTimer perf_step_timer_##metric( \
&(perf_context.metric), env, true, \
(bool)perf_bit_field.enable_##metric##_bit); \
perf_step_timer_##metric.Start();

#define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition, stats, \
ticker_type) \
PerfStepTimer perf_step_timer_##metric(&(perf_context.metric), nullptr, \
false, PerfLevel::kEnableTime, stats, \
ticker_type); \
if (condition) { \
perf_step_timer_##metric.Start(); \
#define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition, stats, \
ticker_type) \
PerfStepTimer perf_step_timer_##metric( \
&(perf_context.metric), nullptr, false, \
(bool)perf_bit_field.enable_##metric##_bit, stats, ticker_type); \
if (condition) { \
perf_step_timer_##metric.Start(); \
}

// Update metric with time elapsed since last START. start time is reset
// to current timestamp.
#define PERF_TIMER_MEASURE(metric) perf_step_timer_##metric.Measure();

// Increase metric value
#define PERF_COUNTER_ADD(metric, value) \
if (perf_level >= PerfLevel::kEnableCount) { \
perf_context.metric += value; \
#define PERF_COUNTER_ADD(metric, value) \
if (perf_bit_field.enable_##metric##_bit) { \
perf_context.metric += value; \
}

// Increase metric value
#define PERF_COUNTER_BY_LEVEL_ADD(metric, value, level) \
if (perf_level >= PerfLevel::kEnableCount && \
perf_context.per_level_perf_context_enabled && \
perf_context.level_to_perf_context) { \
if ((*(perf_context.level_to_perf_context)).find(level) != \
(*(perf_context.level_to_perf_context)).end()) { \
(*(perf_context.level_to_perf_context))[level].metric += value; \
} \
else { \
PerfContextByLevel empty_context; \
(*(perf_context.level_to_perf_context))[level] = empty_context; \
(*(perf_context.level_to_perf_context))[level].metric += value; \
} \
} \
// TODO: 添加一个flag count
LemonHX marked this conversation as resolved.
Show resolved Hide resolved
#define PERF_COUNTER_BY_LEVEL_ADD(metric, value, level) \
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to change this part?

if (perf_context.per_level_perf_context_enabled && \
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems that there is an optimization in the original implementation that it checks whether the perf is enabled. You should check it as well I think, considering that checking perf flags should be faster than the xxx.find() below.

perf_context.level_to_perf_context) { \
if ((*(perf_context.level_to_perf_context)).find(level) != \
(*(perf_context.level_to_perf_context)).end()) { \
(*(perf_context.level_to_perf_context))[level].metric += value; \
} else { \
PerfContextByLevel empty_context; \
(*(perf_context.level_to_perf_context))[level] = empty_context; \
(*(perf_context.level_to_perf_context))[level].metric += value; \
} \
}

#endif

}
} // namespace rocksdb
35 changes: 24 additions & 11 deletions monitoring/perf_level.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,38 @@
// (found in the LICENSE.Apache file in the root directory).
//

#include "rocksdb/perf_level.h"

#include <assert.h>
#include "monitoring/perf_level_imp.h"

namespace rocksdb {
#include "monitoring/perf_level_by_bitfield_imp.h"

#ifdef ROCKSDB_SUPPORT_THREAD_LOCAL
__thread PerfLevel perf_level = kEnableCount;
#else
PerfLevel perf_level = kEnableCount;
#endif
namespace rocksdb {

void SetPerfLevel(PerfLevel level) {
// TODO: discuss keep assertions or not
assert(level > kUninitialized);
assert(level < kOutOfBounds);
perf_level = level;
switch (level) {
case kEnableCount:
perf_bit_field = BitFieldEnableCount;
break;
case kEnableTimeExceptForMutex:
perf_bit_field = BitFieldEnableTimeExceptForMutex;
break;
case kEnableTimeAndCPUTimeExceptForMutex:
perf_bit_field = BitFieldEnableTimeAndCPUTimeExceptForMutex;
break;
case kEnableTime:
perf_bit_field = BitFieldEnableTime;
break;
default:
perf_bit_field = {};
break;
}
perf_bit_field.perf_level = level;
}

PerfLevel GetPerfLevel() {
return perf_level;
}
PerfLevel GetPerfLevel() { return (PerfLevel)perf_bit_field.perf_level; }
LemonHX marked this conversation as resolved.
Show resolved Hide resolved

} // namespace rocksdb
Loading