From e17fd497f55f88b13538bf14415727903d7e72dd Mon Sep 17 00:00:00 2001 From: koarz Date: Fri, 13 Jun 2025 18:32:03 +0800 Subject: [PATCH 01/23] [enhance](meta-service)add bvar for ms kv read write count --- cloud/src/common/bvars.cpp | 178 ++++ cloud/src/common/bvars.h | 118 +++ cloud/src/common/stats.h | 33 + cloud/src/meta-service/mem_txn_kv.cpp | 14 +- cloud/src/meta-service/mem_txn_kv.h | 8 + cloud/src/meta-service/meta_service.cpp | 114 ++- cloud/src/meta-service/meta_service.h | 13 +- cloud/src/meta-service/meta_service_helper.h | 8 + cloud/src/meta-service/meta_service_job.cpp | 3 +- .../meta-service/meta_service_partition.cpp | 6 - .../meta-service/meta_service_resource.cpp | 17 - .../src/meta-service/meta_service_schema.cpp | 2 + .../meta_service_tablet_stats.cpp | 8 +- .../meta-service/meta_service_tablet_stats.h | 1 + cloud/src/meta-service/meta_service_txn.cpp | 14 +- cloud/src/meta-service/txn_kv.cpp | 3 + cloud/src/meta-service/txn_kv.h | 9 + cloud/test/rpc_kv_bvar_test.cpp | 949 ++++++++++++++++++ 18 files changed, 1423 insertions(+), 75 deletions(-) create mode 100644 cloud/src/common/stats.h create mode 100644 cloud/test/rpc_kv_bvar_test.cpp diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 02fc366f83ef43..abedc38e77c58a 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -17,6 +17,8 @@ #include "common/bvars.h" +#include + #include #include @@ -186,4 +188,180 @@ BvarStatusWithTag g_bvar_inverted_checker_abnormal_delete_bitmaps("chec BvarStatusWithTag g_bvar_inverted_checker_delete_bitmaps_scanned("checker", "delete_bitmap_keys_scanned"); BvarStatusWithTag g_bvar_max_rowsets_with_useless_delete_bitmap_version("checker", "max_rowsets_with_useless_delete_bitmap_version"); +// rpc kv rw count +// get_rowset +bvar::Adder g_bvar_rpc_kv_get_rowset_read_counter("rpc_kv","get_rowset_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_rowset_write_counter("rpc_kv","get_rowset_write_counter"); +// get_version +bvar::Adder g_bvar_rpc_kv_get_version_read_counter("rpc_kv","get_version_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_version_write_counter("rpc_kv","get_version_write_counter"); +// get_schema_dict +bvar::Adder g_bvar_rpc_kv_get_schema_dict_read_counter("rpc_kv","get_schema_dict_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_schema_dict_write_counter("rpc_kv","get_schema_dict_write_counter"); +// create_tablets +bvar::Adder g_bvar_rpc_kv_create_tablets_read_counter("rpc_kv","create_tablets_read_counter"); +bvar::Adder g_bvar_rpc_kv_create_tablets_write_counter("rpc_kv","create_tablets_write_counter"); +// update_tablet +bvar::Adder g_bvar_rpc_kv_update_tablet_read_counter("rpc_kv","update_tablet_read_counter"); +bvar::Adder g_bvar_rpc_kv_update_tablet_write_counter("rpc_kv","update_tablet_write_counter"); +// update_tablet_schema +bvar::Adder g_bvar_rpc_kv_update_tablet_schema_read_counter("rpc_kv","update_tablet_schema_read_counter"); +bvar::Adder g_bvar_rpc_kv_update_tablet_schema_write_counter("rpc_kv","update_tablet_schema_write_counter"); +// get_tablet +bvar::Adder g_bvar_rpc_kv_get_tablet_read_counter("rpc_kv","get_tablet_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_tablet_write_counter("rpc_kv","get_tablet_write_counter"); +// prepare_rowset +bvar::Adder g_bvar_rpc_kv_prepare_rowset_read_counter("rpc_kv","prepare_rowset_read_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_rowset_write_counter("rpc_kv","prepare_rowset_write_counter"); +// commit_rowset +bvar::Adder g_bvar_rpc_kv_commit_rowset_read_counter("rpc_kv","commit_rowset_read_counter"); +bvar::Adder g_bvar_rpc_kv_commit_rowset_write_counter("rpc_kv","commit_rowset_write_counter"); +// update_tmp_rowset +bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_read_counter("rpc_kv","update_tmp_rowset_read_counter"); +bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_write_counter("rpc_kv","update_tmp_rowset_write_counter"); +// get_tablet_stats +bvar::Adder g_bvar_rpc_kv_get_tablet_stats_read_counter("rpc_kv","get_tablet_stats_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_tablet_stats_write_counter("rpc_kv","get_tablet_stats_write_counter"); +// update_delete_bitmap +bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_read_counter("rpc_kv","update_delete_bitmap_read_counter"); +bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_write_counter("rpc_kv","update_delete_bitmap_write_counter"); +// get_delete_bitmap +bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_read_counter("rpc_kv","get_delete_bitmap_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_write_counter("rpc_kv","get_delete_bitmap_write_counter"); +// get_delete_bitmap_update_lock +bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_read_counter("rpc_kv","get_delete_bitmap_update_lock_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_write_counter("rpc_kv","get_delete_bitmap_update_lock_write_counter"); +// remove_delete_bitmap_update_lock +bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_read_counter("rpc_kv","remove_delete_bitmap_update_lock_read_counter"); +bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_write_counter("rpc_kv","remove_delete_bitmap_update_lock_write_counter"); +// remove_delete_bitmap +bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_read_counter("rpc_kv","remove_delete_bitmap_read_counter"); +bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_write_counter("rpc_kv","remove_delete_bitmap_write_counter"); +// start_tablet_job +bvar::Adder g_bvar_rpc_kv_start_tablet_job_read_counter("rpc_kv","start_tablet_job_read_counter"); +bvar::Adder g_bvar_rpc_kv_start_tablet_job_write_counter("rpc_kv","start_tablet_job_write_counter"); +// finish_tablet_job +bvar::Adder g_bvar_rpc_kv_finish_tablet_job_read_counter("rpc_kv","finish_tablet_job_read_counter"); +bvar::Adder g_bvar_rpc_kv_finish_tablet_job_write_counter("rpc_kv","finish_tablet_job_write_counter"); +// prepare_index +bvar::Adder g_bvar_rpc_kv_prepare_index_read_counter("rpc_kv","prepare_index_read_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_index_write_counter("rpc_kv","prepare_index_write_counter"); +// commit_index +bvar::Adder g_bvar_rpc_kv_commit_index_read_counter("rpc_kv","commit_index_read_counter"); +bvar::Adder g_bvar_rpc_kv_commit_index_write_counter("rpc_kv","commit_index_write_counter"); +// drop_index +bvar::Adder g_bvar_rpc_kv_drop_index_read_counter("rpc_kv","drop_index_read_counter"); +bvar::Adder g_bvar_rpc_kv_drop_index_write_counter("rpc_kv","drop_index_write_counter"); +// prepare_partition +bvar::Adder g_bvar_rpc_kv_prepare_partition_read_counter("rpc_kv","prepare_partition_read_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_partition_write_counter("rpc_kv","prepare_partition_write_counter"); +// commit_partition +bvar::Adder g_bvar_rpc_kv_commit_partition_read_counter("rpc_kv","commit_partition_read_counter"); +bvar::Adder g_bvar_rpc_kv_commit_partition_write_counter("rpc_kv","commit_partition_write_counter"); +// drop_partition +bvar::Adder g_bvar_rpc_kv_drop_partition_read_counter("rpc_kv","drop_partition_read_counter"); +bvar::Adder g_bvar_rpc_kv_drop_partition_write_counter("rpc_kv","drop_partition_write_counter"); +// check_kv +bvar::Adder g_bvar_rpc_kv_check_kv_read_counter("rpc_kv","check_kv_read_counter"); +bvar::Adder g_bvar_rpc_kv_check_kv_write_counter("rpc_kv","check_kv_write_counter"); +// get_obj_store_info +bvar::Adder g_bvar_rpc_kv_get_obj_store_info_read_counter("rpc_kv","get_obj_store_info_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_obj_store_info_write_counter("rpc_kv","get_obj_store_info_write_counter"); +// alter_storage_vault +bvar::Adder g_bvar_rpc_kv_alter_storage_vault_read_counter("rpc_kv","alter_storage_vault_read_counter"); +bvar::Adder g_bvar_rpc_kv_alter_storage_vault_write_counter("rpc_kv","alter_storage_vault_write_counter"); +// alter_obj_store_info +bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_read_counter("rpc_kv","alter_obj_store_info_read_counter"); +bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_write_counter("rpc_kv","alter_obj_store_info_write_counter"); +// update_ak_sk +bvar::Adder g_bvar_rpc_kv_update_ak_sk_read_counter("rpc_kv","update_ak_sk_read_counter"); +bvar::Adder g_bvar_rpc_kv_update_ak_sk_write_counter("rpc_kv","update_ak_sk_write_counter"); +// create_instance +bvar::Adder g_bvar_rpc_kv_create_instance_read_counter("rpc_kv","create_instance_read_counter"); +bvar::Adder g_bvar_rpc_kv_create_instance_write_counter("rpc_kv","create_instance_write_counter"); +// get_instance +bvar::Adder g_bvar_rpc_kv_get_instance_read_counter("rpc_kv","get_instance_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_instance_write_counter("rpc_kv","get_instance_write_counter"); +// alter_cluster +bvar::Adder g_bvar_rpc_kv_alter_cluster_read_counter("rpc_kv","alter_cluster_read_counter"); +bvar::Adder g_bvar_rpc_kv_alter_cluster_write_counter("rpc_kv","alter_cluster_write_counter"); +// get_cluster +bvar::Adder g_bvar_rpc_kv_get_cluster_read_counter("rpc_kv","get_cluster_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_cluster_write_counter("rpc_kv","get_cluster_write_counter"); +// create_stage +bvar::Adder g_bvar_rpc_kv_create_stage_read_counter("rpc_kv","create_stage_read_counter"); +bvar::Adder g_bvar_rpc_kv_create_stage_write_counter("rpc_kv","create_stage_write_counter"); +// get_stage +bvar::Adder g_bvar_rpc_kv_get_stage_read_counter("rpc_kv","get_stage_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_stage_write_counter("rpc_kv","get_stage_write_counter"); +// get_iam +bvar::Adder g_bvar_rpc_kv_get_iam_read_counter("rpc_kv","get_iam_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_iam_write_counter("rpc_kv","get_iam_write_counter"); +// alter_iam +bvar::Adder g_bvar_rpc_kv_alter_iam_read_counter("rpc_kv","alter_iam_read_counter"); +bvar::Adder g_bvar_rpc_kv_alter_iam_write_counter("rpc_kv","alter_iam_write_counter"); +// alter_ram_user +bvar::Adder g_bvar_rpc_kv_alter_ram_user_read_counter("rpc_kv","alter_ram_user_read_counter"); +bvar::Adder g_bvar_rpc_kv_alter_ram_user_write_counter("rpc_kv","alter_ram_user_write_counter"); +// begin_copy +bvar::Adder g_bvar_rpc_kv_begin_copy_read_counter("rpc_kv","begin_copy_read_counter"); +bvar::Adder g_bvar_rpc_kv_begin_copy_write_counter("rpc_kv","begin_copy_write_counter"); +// finish_copy +bvar::Adder g_bvar_rpc_kv_finish_copy_read_counter("rpc_kv","finish_copy_read_counter"); +bvar::Adder g_bvar_rpc_kv_finish_copy_write_counter("rpc_kv","finish_copy_write_counter"); +// get_copy_job +bvar::Adder g_bvar_rpc_kv_get_copy_job_read_counter("rpc_kv","get_copy_job_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_copy_job_write_counter("rpc_kv","get_copy_job_write_counter"); +// get_copy_files +bvar::Adder g_bvar_rpc_kv_get_copy_files_read_counter("rpc_kv","get_copy_files_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_copy_files_write_counter("rpc_kv","get_copy_files_write_counter"); +// filter_copy_files +bvar::Adder g_bvar_rpc_kv_filter_copy_files_read_counter("rpc_kv","filter_copy_files_read_counter"); +bvar::Adder g_bvar_rpc_kv_filter_copy_files_write_counter("rpc_kv","filter_copy_files_write_counter"); +// get_cluster_status +bvar::Adder g_bvar_rpc_kv_get_cluster_status_read_counter("rpc_kv","get_cluster_status_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_cluster_status_write_counter("rpc_kv","get_cluster_status_write_counter"); +// begin_txn +bvar::Adder g_bvar_rpc_kv_begin_txn_read_counter("rpc_kv","begin_txn_read_counter"); +bvar::Adder g_bvar_rpc_kv_begin_txn_write_counter("rpc_kv","begin_txn_write_counter"); +// precommit_txn +bvar::Adder g_bvar_rpc_kv_precommit_txn_read_counter("rpc_kv","precommit_txn_read_counter"); +bvar::Adder g_bvar_rpc_kv_precommit_txn_write_counter("rpc_kv","precommit_txn_write_counter"); +// get_rl_task_commit_attach +bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_read_counter("rpc_kv","get_rl_task_commit_attach_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_write_counter("rpc_kv","get_rl_task_commit_attach_write_counter"); +// reset_rl_progress +bvar::Adder g_bvar_rpc_kv_reset_rl_progress_read_counter("rpc_kv","reset_rl_progress_read_counter"); +bvar::Adder g_bvar_rpc_kv_reset_rl_progress_write_counter("rpc_kv","reset_rl_progress_write_counter"); +// commit_txn +bvar::Adder g_bvar_rpc_kv_commit_txn_read_counter("rpc_kv","commit_txn_read_counter"); +bvar::Adder g_bvar_rpc_kv_commit_txn_write_counter("rpc_kv","commit_txn_write_counter"); +// abort_txn +bvar::Adder g_bvar_rpc_kv_abort_txn_read_counter("rpc_kv","abort_txn_read_counter"); +bvar::Adder g_bvar_rpc_kv_abort_txn_write_counter("rpc_kv","abort_txn_write_counter"); +// get_txn +bvar::Adder g_bvar_rpc_kv_get_txn_read_counter("rpc_kv","get_txn_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_txn_write_counter("rpc_kv","get_txn_write_counter"); +// get_current_max_txn_id +bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_read_counter("rpc_kv","get_current_max_txn_id_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_write_counter("rpc_kv","get_current_max_txn_id_write_counter"); +// begin_sub_txn +bvar::Adder g_bvar_rpc_kv_begin_sub_txn_read_counter("rpc_kv","begin_sub_txn_read_counter"); +bvar::Adder g_bvar_rpc_kv_begin_sub_txn_write_counter("rpc_kv","begin_sub_txn_write_counter"); +// abort_sub_txn +bvar::Adder g_bvar_rpc_kv_abort_sub_txn_read_counter("rpc_kv","abort_sub_txn_read_counter"); +bvar::Adder g_bvar_rpc_kv_abort_sub_txn_write_counter("rpc_kv","abort_sub_txn_write_counter"); +// abort_txn_with_coordinator +bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_read_counter("rpc_kv","abort_txn_with_coordinator_read_counter"); +bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_write_counter("rpc_kv","abort_txn_with_coordinator_write_counter"); +// check_txn_conflict +bvar::Adder g_bvar_rpc_kv_check_txn_conflict_read_counter("rpc_kv","check_txn_conflict_read_counter"); +bvar::Adder g_bvar_rpc_kv_check_txn_conflict_write_counter("rpc_kv","check_txn_conflict_write_counter"); +// clean_txn_label +bvar::Adder g_bvar_rpc_kv_clean_txn_label_read_counter("rpc_kv","clean_txn_label_read_counter"); +bvar::Adder g_bvar_rpc_kv_clean_txn_label_write_counter("rpc_kv","clean_txn_label_write_counter"); +// get_txn_id +bvar::Adder g_bvar_rpc_kv_get_txn_id_read_counter("rpc_kv","get_txn_id_read_counter"); +bvar::Adder g_bvar_rpc_kv_get_txn_id_write_counter("rpc_kv","get_txn_id_write_counter"); + // clang-format on diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 3f496c58042790..9b0dda69329bec 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -254,3 +254,121 @@ extern BvarStatusWithTag g_bvar_inverted_checker_leaked_delete_bitmaps; extern BvarStatusWithTag g_bvar_inverted_checker_abnormal_delete_bitmaps; extern BvarStatusWithTag g_bvar_inverted_checker_delete_bitmaps_scanned; extern BvarStatusWithTag g_bvar_max_rowsets_with_useless_delete_bitmap_version; + +// rpc kv +extern bvar::Adder g_bvar_rpc_kv_get_rowset_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_rowset_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_version_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_version_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_write_counter; +extern bvar::Adder g_bvar_rpc_kv_create_tablets_read_counter; +extern bvar::Adder g_bvar_rpc_kv_create_tablets_write_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tablet_read_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tablet_write_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_read_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_tablet_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_tablet_write_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_read_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_write_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_rowset_read_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_rowset_write_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_read_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_write_counter; +extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_read_counter; +extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_write_counter; +extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_read_counter; +extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_write_counter; +extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_read_counter; +extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_write_counter; +extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_read_counter; +extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_write_counter; +extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_read_counter; +extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_write_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_index_read_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_index_write_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_index_read_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_index_write_counter; +extern bvar::Adder g_bvar_rpc_kv_drop_index_read_counter; +extern bvar::Adder g_bvar_rpc_kv_drop_index_write_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_partition_read_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_partition_write_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_partition_read_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_partition_write_counter; +extern bvar::Adder g_bvar_rpc_kv_drop_partition_read_counter; +extern bvar::Adder g_bvar_rpc_kv_drop_partition_write_counter; +extern bvar::Adder g_bvar_rpc_kv_check_kv_read_counter; +extern bvar::Adder g_bvar_rpc_kv_check_kv_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_write_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_read_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_write_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_read_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_write_counter; +extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_read_counter; +extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_write_counter; +extern bvar::Adder g_bvar_rpc_kv_create_instance_read_counter; +extern bvar::Adder g_bvar_rpc_kv_create_instance_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_instance_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_instance_write_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_cluster_read_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_cluster_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_cluster_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_cluster_write_counter; +extern bvar::Adder g_bvar_rpc_kv_create_stage_read_counter; +extern bvar::Adder g_bvar_rpc_kv_create_stage_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_stage_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_stage_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_iam_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_iam_write_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_iam_read_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_iam_write_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_read_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_write_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_copy_read_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_copy_write_counter; +extern bvar::Adder g_bvar_rpc_kv_finish_copy_read_counter; +extern bvar::Adder g_bvar_rpc_kv_finish_copy_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_copy_job_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_copy_job_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_copy_files_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_copy_files_write_counter; +extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_read_counter; +extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_write_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_txn_read_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_txn_write_counter; +extern bvar::Adder g_bvar_rpc_kv_precommit_txn_read_counter; +extern bvar::Adder g_bvar_rpc_kv_precommit_txn_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_write_counter; +extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_read_counter; +extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_write_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_txn_read_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_txn_write_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_txn_read_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_txn_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_txn_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_txn_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_write_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_read_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_write_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_read_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_write_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_read_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_write_counter; +extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_read_counter; +extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_write_counter; +extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_read_counter; +extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_txn_id_read_counter; +extern bvar::Adder g_bvar_rpc_kv_get_txn_id_write_counter; diff --git a/cloud/src/common/stats.h b/cloud/src/common/stats.h new file mode 100644 index 00000000000000..b9ff4e3a325cac --- /dev/null +++ b/cloud/src/common/stats.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +namespace doris::cloud { +struct KVStats { + KVStats(bvar::Adder& read_counter, bvar::Adder& write_counter) + : read_counter(read_counter), write_counter(write_counter) {} + + bvar::Adder& read_counter; + bvar::Adder& write_counter; +}; + +} // namespace doris::cloud \ No newline at end of file diff --git a/cloud/src/meta-service/mem_txn_kv.cpp b/cloud/src/meta-service/mem_txn_kv.cpp index 84593de3c200f8..7eef0a5dd50a90 100644 --- a/cloud/src/meta-service/mem_txn_kv.cpp +++ b/cloud/src/meta-service/mem_txn_kv.cpp @@ -248,6 +248,7 @@ void Transaction::put(std::string_view key, std::string_view val) { writes_.insert_or_assign(k, v); op_list_.emplace_back(ModifyOpType::PUT, k, v); ++num_put_keys_; + kv_->write_count_++; put_bytes_ += key.size() + val.size(); approximate_bytes_ += key.size() + val.size(); } @@ -300,6 +301,8 @@ TxnErrorCode Transaction::inner_get(const std::string& key, std::string* val, bo return TxnErrorCode::TXN_KEY_NOT_FOUND; } } + num_get_keys_++; + kv_->read_count_++; return TxnErrorCode::TXN_OK; } @@ -344,12 +347,15 @@ TxnErrorCode Transaction::inner_get(const std::string& begin, const std::string& } std::vector> kv_list(kv_map.begin(), kv_map.end()); + num_get_keys_ += kv_list.size(); + kv_->read_count_ += kv_list.size(); *iter = std::make_unique(std::move(kv_list), more); return TxnErrorCode::TXN_OK; } void Transaction::atomic_set_ver_key(std::string_view key_prefix, std::string_view val) { std::lock_guard l(lock_); + kv_->write_count_++; std::string k(key_prefix.data(), key_prefix.size()); std::string v(val.data(), val.size()); unreadable_keys_.insert(k); @@ -362,6 +368,7 @@ void Transaction::atomic_set_ver_key(std::string_view key_prefix, std::string_vi void Transaction::atomic_set_ver_value(std::string_view key, std::string_view value) { std::lock_guard l(lock_); + kv_->write_count_++; std::string k(key.data(), key.size()); std::string v(value.data(), value.size()); unreadable_keys_.insert(k); @@ -377,6 +384,7 @@ void Transaction::atomic_add(std::string_view key, int64_t to_add) { std::string v(sizeof(to_add), '\0'); memcpy(v.data(), &to_add, sizeof(to_add)); std::lock_guard l(lock_); + kv_->write_count_++; op_list_.emplace_back(ModifyOpType::ATOMIC_ADD, std::move(k), std::move(v)); ++num_put_keys_; @@ -395,6 +403,7 @@ bool Transaction::decode_atomic_int(std::string_view data, int64_t* val) { void Transaction::remove(std::string_view key) { std::lock_guard l(lock_); + kv_->write_count_++; std::string k(key.data(), key.size()); writes_.erase(k); std::string end_key = k; @@ -421,7 +430,9 @@ void Transaction::remove(std::string_view begin, std::string_view end) { remove_ranges_.emplace_back(begin_k, end_k); op_list_.emplace_back(ModifyOpType::REMOVE_RANGE, begin_k, end_k); } - ++num_del_keys_; + kv_->write_count_ += 2; + // same as normal txn + num_del_keys_ += 2; delete_bytes_ += begin.size() + end.size(); approximate_bytes_ += begin.size() + end.size(); } @@ -480,6 +491,7 @@ TxnErrorCode Transaction::batch_get(std::vector>* res auto ret = inner_get(k, &val, opts.snapshot); ret == TxnErrorCode::TXN_OK ? res->push_back(val) : res->push_back(std::nullopt); } + num_get_keys_ += keys.size(); return TxnErrorCode::TXN_OK; } diff --git a/cloud/src/meta-service/mem_txn_kv.h b/cloud/src/meta-service/mem_txn_kv.h index 63fb008f586ed8..9f53fed2c792a9 100644 --- a/cloud/src/meta-service/mem_txn_kv.h +++ b/cloud/src/meta-service/mem_txn_kv.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -29,6 +30,7 @@ #include #include +#include "common/stats.h" #include "meta-service/txn_kv_error.h" #include "txn_kv.h" @@ -57,6 +59,9 @@ class MemTxnKv : public TxnKv, public std::enable_shared_from_this { TxnErrorCode get_kv(const std::string& begin, const std::string& end, int64_t version, int limit, bool* more, std::map* kv_list); + int64_t read_count_ {}; + int64_t write_count_ {}; + private: using OpTuple = std::tuple; TxnErrorCode update(const std::set& read_set, const std::vector& op_list, @@ -189,6 +194,8 @@ class Transaction : public cloud::Transaction { size_t approximate_bytes() const override { return approximate_bytes_; } + size_t num_get_keys() const override { return num_get_keys_; } + size_t num_del_keys() const override { return num_del_keys_; } size_t num_put_keys() const override { return num_put_keys_; } @@ -218,6 +225,7 @@ class Transaction : public cloud::Transaction { int64_t read_version_ = -1; size_t approximate_bytes_ {0}; + size_t num_get_keys_ {0}; size_t num_del_keys_ {0}; size_t num_put_keys_ {0}; size_t delete_bytes_ {0}; diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 3dcf166c0b2b82..39214b0d0e54df 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,7 @@ #include "common/config.h" #include "common/encryption_util.h" #include "common/logging.h" +#include "common/stats.h" #include "common/stopwatch.h" #include "common/string_util.h" #include "common/util.h" @@ -90,7 +92,9 @@ std::string get_instance_id(const std::shared_ptr& rc_mgr, std::vector nodes; std::string err = rc_mgr->get_node(cloud_unique_id, &nodes); - { TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); } + { + TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); + } std::string instance_id; if (!err.empty()) { // cache can't find cloud_unique_id, so degraded by parse cloud_unique_id @@ -250,7 +254,7 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, } code = MetaServiceCode::OK; - std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { msg = "failed to create txn"; @@ -287,7 +291,9 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, response->set_version(version_pb.version()); response->add_version_update_time_ms(version_pb.update_time_ms()); } - { TEST_SYNC_POINT_CALLBACK("get_version_code", &code); } + { + TEST_SYNC_POINT_CALLBACK("get_version_code", &code); + } return; } else if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { msg = "not found"; @@ -358,7 +364,10 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr code = cast_as(err); break; } - + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.read_counter << txn->num_get_keys(); + stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + }); for (size_t i = response->versions_size(); i < num_acquired; i += BATCH_SIZE) { size_t limit = (i + BATCH_SIZE < num_acquired) ? i + BATCH_SIZE : num_acquired; version_keys.clear(); @@ -433,7 +442,7 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode& code, std::string& msg, const doris::TabletMetaCloudPB& meta, std::shared_ptr txn_kv, const std::string& instance_id, - std::set>& saved_schema) { + std::set>& saved_schema, KVStats& stats) { doris::TabletMetaCloudPB tablet_meta(meta); bool has_first_rowset = tablet_meta.rs_metas_size() > 0; @@ -456,6 +465,10 @@ void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode msg = "failed to init txn"; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.read_counter << txn->num_get_keys(); + stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + }); std::string rs_key, rs_val; if (has_first_rowset) { @@ -593,6 +606,10 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll msg = fmt::format("failed to create txn"); return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.read_counter << txn0->num_get_keys(); + stats.write_counter << txn0->num_del_keys() + txn0->num_put_keys(); + }); InstanceKeyInfo key_info {instance_id}; std::string key; @@ -652,7 +669,8 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll std::set> saved_schema; TEST_SYNC_POINT_RETURN_WITH_VOID("create_tablets"); for (auto& tablet_meta : request->tablet_metas()) { - internal_create_tablet(request, code, msg, tablet_meta, txn_kv_, instance_id, saved_schema); + internal_create_tablet(request, code, msg, tablet_meta, txn_kv_, instance_id, saved_schema, + stats); if (code != MetaServiceCode::OK) { return; } @@ -733,7 +751,7 @@ void MetaServiceImpl::update_tablet(::google::protobuf::RpcController* controlle return; } RPC_RATE_LIMIT(update_tablet) - std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -840,7 +858,6 @@ void MetaServiceImpl::update_tablet_schema(::google::protobuf::RpcController* co RPC_RATE_LIMIT(update_tablet_schema) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -911,7 +928,7 @@ void MetaServiceImpl::get_tablet(::google::protobuf::RpcController* controller, return; } RPC_RATE_LIMIT(get_tablet) - std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1000,6 +1017,7 @@ static void fill_schema_from_dict(MetaServiceCode& code, std::string& msg, * @param txn_id The transaction ID to check. * @param code Reference to the error code to be set in case of failure. * @param msg Reference to the error message to be set in case of failure. +* @param stats count rpc kv read write num * @return true if the transaction status matches the expected status, false otherwise. */ static bool check_transaction_status(TxnStatusPB expect_status, Transaction* txn, @@ -1091,7 +1109,7 @@ void MetaServiceImpl::prepare_rowset(::google::protobuf::RpcController* controll int64_t tablet_id = rowset_meta.tablet_id(); const auto& rowset_id = rowset_meta.rowset_id_v2(); auto tmp_rs_key = meta_rowset_tmp_key({instance_id, rowset_meta.txn_id(), tablet_id}); - std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1228,7 +1246,6 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle auto tmp_rs_key = meta_rowset_tmp_key({instance_id, rowset_meta.txn_id(), tablet_id}); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1383,7 +1400,6 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr MetaRowsetTmpKeyInfo key_info {instance_id, txn_id, tablet_id}; meta_rowset_tmp_key(key_info, &update_key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1648,7 +1664,10 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, LOG(WARNING) << msg; return; } - + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.read_counter << txn->num_get_keys(); + stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + }); TabletIndexPB idx; // Get tablet id index from kv get_tablet_idx(code, msg, txn.get(), instance_id, tablet_id, idx); @@ -1810,7 +1829,6 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro } RPC_RATE_LIMIT(get_tablet_stats) - std::unique_ptr txn; for (auto& i : request->tablet_idx()) { TabletIndexPB idx(i); // FIXME(plat1ko): Get all tablet stats in one txn @@ -1820,6 +1838,11 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro msg = fmt::format("failed to create txn, tablet_id={}", idx.tablet_id()); return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.read_counter << txn->num_get_keys(); + stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + txn.reset(nullptr); + }); if (!(/* idx.has_db_id() && */ idx.has_table_id() && idx.has_index_id() && idx.has_partition_id() && i.has_tablet_id())) { get_tablet_idx(code, msg, txn.get(), instance_id, idx.tablet_id(), idx); @@ -2130,7 +2153,6 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont auto table_id = request->table_id(); auto tablet_id = request->tablet_id(); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2266,6 +2288,8 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont err = txn->commit(); TEST_SYNC_POINT_CALLBACK("update_delete_bitmap:commit:err", request->initiator(), i, &err); + stats.read_counter << txn->num_get_keys(); + stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); total_txn_put_keys += txn->num_put_keys(); total_txn_put_bytes += txn->put_bytes(); total_txn_size += txn->approximate_bytes(); @@ -2307,7 +2331,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont #ifdef ENABLE_INJECTION_POINT if (config::enable_update_delete_bitmap_kv_check) { - if (!check_delete_bitmap_kv_exists(code, msg, txn, key, instance_id, tablet_id, + if (!check_delete_bitmap_kv_exists(code, msg, txn0, key, instance_id, tablet_id, request->lock_id())) { return; } @@ -2489,6 +2513,10 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control msg = "failed to init txn"; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.read_counter << txn->num_get_keys(); + stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + }); MetaDeleteBitmapInfo start_key_info {instance_id, tablet_id, rowset_ids[i], begin_versions[i], 0}; MetaDeleteBitmapInfo end_key_info {instance_id, tablet_id, rowset_ids[i], end_versions[i], @@ -2606,6 +2634,10 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control msg = "failed to init txn"; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.read_counter << txn->num_get_keys(); + stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + }); TabletIndexPB idx(request->idx()); TabletStatsPB tablet_stat; internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat, @@ -2672,7 +2704,7 @@ bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std:: const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, std::string& instance_id, std::string& lock_key, - std::string lock_use_version) { + std::string lock_use_version, KVStats* stats) { bool require_tablet_stats = request->has_require_compaction_stats() ? request->require_compaction_stats() : false; if (!require_tablet_stats) return true; @@ -2692,6 +2724,10 @@ bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std:: msg = "failed to init txn"; return false; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats->read_counter << txn->num_get_keys(); + stats->write_counter << txn->num_del_keys() + txn->num_put_keys(); + }); auto table_id = request->table_id(); std::stringstream ss; if (!config::enable_batch_get_mow_tablet_stats_and_meta) { @@ -2866,7 +2902,8 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2( google::protobuf::RpcController* controller, const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, - std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss) { + std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss, + KVStats* stats) { VLOG_DEBUG << "get delete bitmap update lock in v2 for table=" << request->table_id() << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator(); auto table_id = request->table_id(); @@ -2883,6 +2920,10 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2( msg = "failed to init txn"; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats->read_counter << txn->num_get_keys(); + stats->write_counter << txn->num_del_keys() + txn->num_put_keys(); + }); std::string lock_val; DeleteBitmapUpdateLockPB lock_info; err = txn->get(lock_key, &lock_val); @@ -3095,7 +3136,8 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2( } } - if (!get_mow_tablet_stats_and_meta(code, msg, request, response, instance_id, lock_key, "v2")) { + if (!get_mow_tablet_stats_and_meta(code, msg, request, response, instance_id, lock_key, "v2", + stats)) { return; } } @@ -3104,7 +3146,8 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v1( google::protobuf::RpcController* controller, const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, - std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss) { + std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss, + KVStats* stats) { VLOG_DEBUG << "get delete bitmap update lock in v1 for table=" << request->table_id() << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator(); std::unique_ptr txn; @@ -3114,6 +3157,10 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v1( msg = "failed to init txn"; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats->read_counter << txn->num_get_keys(); + stats->write_counter << txn->num_del_keys() + txn->num_put_keys(); + }); auto table_id = request->table_id(); std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}); std::string lock_val; @@ -3179,7 +3226,8 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v1( return; } - if (!get_mow_tablet_stats_and_meta(code, msg, request, response, instance_id, lock_key, "v1")) { + if (!get_mow_tablet_stats_and_meta(code, msg, request, response, instance_id, lock_key, "v1", + stats)) { return; }; } @@ -3188,7 +3236,8 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v2( google::protobuf::RpcController* controller, const RemoveDeleteBitmapUpdateLockRequest* request, RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, - std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss) { + std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss, + KVStats* stats) { VLOG_DEBUG << "remove delete bitmap update lock in v2 for table=" << request->table_id() << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator(); std::unique_ptr txn; @@ -3198,6 +3247,10 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v2( msg = "failed to init txn"; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats->read_counter << txn->num_get_keys(); + stats->write_counter << txn->num_del_keys() + txn->num_put_keys(); + }); if (request->lock_id() == COMPACTION_DELETE_BITMAP_LOCK_ID) { std::string tablet_compaction_key = mow_tablet_compaction_key({instance_id, request->table_id(), request->initiator()}); @@ -3267,7 +3320,8 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v1( google::protobuf::RpcController* controller, const RemoveDeleteBitmapUpdateLockRequest* request, RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, - std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss) { + std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss, + KVStats* stats) { VLOG_DEBUG << "remove delete bitmap update lock in v1 for table=" << request->table_id() << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator(); std::unique_ptr txn; @@ -3277,6 +3331,10 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v1( msg = "failed to init txn"; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats->read_counter << txn->num_get_keys(); + stats->write_counter << txn->num_del_keys() + txn->num_put_keys(); + }); std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, request->table_id(), -1}); std::string lock_val; @@ -3353,10 +3411,10 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl << " use_version=" << use_version; if (use_version == "v2") { get_delete_bitmap_update_lock_v2(controller, request, response, done, instance_id, code, - msg, ss); + msg, ss, &stats); } else { get_delete_bitmap_update_lock_v1(controller, request, response, done, instance_id, code, - msg, ss); + msg, ss, &stats); } } @@ -3384,10 +3442,10 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock( << " use_version=" << use_version; if (use_version == "v2") { remove_delete_bitmap_update_lock_v2(controller, request, response, done, instance_id, code, - msg, ss); + msg, ss, &stats); } else { remove_delete_bitmap_update_lock_v1(controller, request, response, done, instance_id, code, - msg, ss); + msg, ss, &stats); } } @@ -3424,7 +3482,6 @@ void MetaServiceImpl::remove_delete_bitmap(google::protobuf::RpcController* cont msg = ss.str(); return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to init txn"; @@ -3563,7 +3620,6 @@ void MetaServiceImpl::get_schema_dict(::google::protobuf::RpcController* control RPC_RATE_LIMIT(get_schema_dict) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index e28872917b5085..5bff9710be5794 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -28,6 +28,7 @@ #include #include "common/config.h" +#include "common/stats.h" #include "cpp/sync_point.h" #include "meta-service/delete_bitmap_lock_white_list.h" #include "meta-service/txn_kv.h" @@ -329,35 +330,37 @@ class MetaServiceImpl : public cloud::MetaService { const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, std::string& instance_id, std::string& lock_key, - std::string lock_use_version); + std::string lock_use_version, KVStats* stats); void get_delete_bitmap_update_lock_v2(google::protobuf::RpcController* controller, const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, - std::string& msg, std::stringstream& ss); + std::string& msg, std::stringstream& ss, KVStats* stats); void get_delete_bitmap_update_lock_v1(google::protobuf::RpcController* controller, const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, - std::string& msg, std::stringstream& ss); + std::string& msg, std::stringstream& ss, KVStats* stats); void remove_delete_bitmap_update_lock_v2(google::protobuf::RpcController* controller, const RemoveDeleteBitmapUpdateLockRequest* request, RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, - std::string& msg, std::stringstream& ss); + std::string& msg, std::stringstream& ss, + KVStats* stats); void remove_delete_bitmap_update_lock_v1(google::protobuf::RpcController* controller, const RemoveDeleteBitmapUpdateLockRequest* request, RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, - std::string& msg, std::stringstream& ss); + std::string& msg, std::stringstream& ss, + KVStats* stats); std::shared_ptr txn_kv_; std::shared_ptr resource_mgr_; diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index cd9ed2f7f1d9fd..cb50aa5cf67086 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -29,6 +29,7 @@ #include "common/bvars.h" #include "common/config.h" #include "common/logging.h" +#include "common/stats.h" #include "common/stopwatch.h" #include "common/util.h" #include "cpp/sync_point.h" @@ -232,9 +233,12 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { brpc::ClosureGuard closure_guard(done); \ [[maybe_unused]] std::stringstream ss; \ [[maybe_unused]] MetaServiceCode code = MetaServiceCode::OK; \ + [[maybe_unused]] std::unique_ptr txn; \ [[maybe_unused]] std::string msg; \ [[maybe_unused]] std::string instance_id; \ [[maybe_unused]] bool drop_request = false; \ + [[maybe_unused]] KVStats stats(g_bvar_rpc_kv_##func_name##_read_counter, \ + g_bvar_rpc_kv_##func_name##_write_counter); \ std::unique_ptr> defer_status((int*)0x01, [&](int*) { \ response->mutable_status()->set_code(code); \ response->mutable_status()->set_msg(msg); \ @@ -242,6 +246,10 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { closure_guard.reset(nullptr); \ if (config::use_detailed_metrics && !instance_id.empty() && !drop_request) { \ g_bvar_ms_##func_name.put(instance_id, sw.elapsed_us()); \ + if (txn != nullptr) { \ + stats.read_counter << txn->num_get_keys(); \ + stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); \ + } \ } \ }); diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index 0ff10ed5a865ea..ac6408f1964722 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -28,6 +28,7 @@ #include "common/bvars.h" #include "common/config.h" #include "common/logging.h" +#include "common/stats.h" #include "common/util.h" #include "cpp/sync_point.h" #include "keys.h" @@ -425,7 +426,6 @@ void MetaServiceImpl::start_tablet_job(::google::protobuf::RpcController* contro return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1495,7 +1495,6 @@ void MetaServiceImpl::finish_tablet_job(::google::protobuf::RpcController* contr } bool need_commit = false; - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); diff --git a/cloud/src/meta-service/meta_service_partition.cpp b/cloud/src/meta-service/meta_service_partition.cpp index d165b8b5e06bb2..ea629fd0f4869f 100644 --- a/cloud/src/meta-service/meta_service_partition.cpp +++ b/cloud/src/meta-service/meta_service_partition.cpp @@ -94,7 +94,6 @@ void MetaServiceImpl::prepare_index(::google::protobuf::RpcController* controlle return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -179,7 +178,6 @@ void MetaServiceImpl::commit_index(::google::protobuf::RpcController* controller return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -263,7 +261,6 @@ void MetaServiceImpl::drop_index(::google::protobuf::RpcController* controller, return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -368,7 +365,6 @@ void MetaServiceImpl::prepare_partition(::google::protobuf::RpcController* contr return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -455,7 +451,6 @@ void MetaServiceImpl::commit_partition(::google::protobuf::RpcController* contro return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -544,7 +539,6 @@ void MetaServiceImpl::drop_partition(::google::protobuf::RpcController* controll return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index eb88694ff5776f..f4a993624c864b 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -226,7 +226,6 @@ void MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -987,7 +986,6 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1255,7 +1253,6 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1415,7 +1412,6 @@ void MetaServiceImpl::update_ak_sk(google::protobuf::RpcController* controller, std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1646,7 +1642,6 @@ void MetaServiceImpl::create_instance(google::protobuf::RpcController* controlle return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1932,7 +1927,6 @@ void MetaServiceImpl::get_instance(google::protobuf::RpcController* controller, std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2443,7 +2437,6 @@ void MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller, std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2591,7 +2584,6 @@ void MetaServiceImpl::create_stage(::google::protobuf::RpcController* controller std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2751,7 +2743,6 @@ void MetaServiceImpl::get_stage(google::protobuf::RpcController* controller, std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3133,7 +3124,6 @@ void MetaServiceImpl::get_iam(google::protobuf::RpcController* controller, std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3222,7 +3212,6 @@ void MetaServiceImpl::alter_iam(google::protobuf::RpcController* controller, std::string key = system_meta_service_arn_info_key(); std::string val; - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3318,7 +3307,6 @@ void MetaServiceImpl::alter_ram_user(google::protobuf::RpcController* controller std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3397,7 +3385,6 @@ void MetaServiceImpl::begin_copy(google::protobuf::RpcController* controller, return; } RPC_RATE_LIMIT(begin_copy) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3515,7 +3502,6 @@ void MetaServiceImpl::finish_copy(google::protobuf::RpcController* controller, } RPC_RATE_LIMIT(finish_copy) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3619,7 +3605,6 @@ void MetaServiceImpl::get_copy_job(google::protobuf::RpcController* controller, return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3672,7 +3657,6 @@ void MetaServiceImpl::get_copy_files(google::protobuf::RpcController* controller } RPC_RATE_LIMIT(get_copy_files) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3736,7 +3720,6 @@ void MetaServiceImpl::filter_copy_files(google::protobuf::RpcController* control } RPC_RATE_LIMIT(filter_copy_files) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); diff --git a/cloud/src/meta-service/meta_service_schema.cpp b/cloud/src/meta-service/meta_service_schema.cpp index bbbedd1e3abe8e..92456691ff1e56 100644 --- a/cloud/src/meta-service/meta_service_schema.cpp +++ b/cloud/src/meta-service/meta_service_schema.cpp @@ -26,10 +26,12 @@ #include #include +#include #include #include "common/config.h" #include "common/logging.h" +#include "common/stats.h" #include "common/util.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" diff --git a/cloud/src/meta-service/meta_service_tablet_stats.cpp b/cloud/src/meta-service/meta_service_tablet_stats.cpp index da556b6c574557..e9229c031ed354 100644 --- a/cloud/src/meta-service/meta_service_tablet_stats.cpp +++ b/cloud/src/meta-service/meta_service_tablet_stats.cpp @@ -27,6 +27,7 @@ #include #include "common/logging.h" +#include "common/stats.h" #include "common/util.h" #include "meta-service/keys.h" #include "meta-service/meta_service.h" @@ -170,10 +171,11 @@ void merge_tablet_stats(TabletStatsPB& stats, const TabletStats& detached_stats) void internal_get_tablet_stats(MetaServiceCode& code, std::string& msg, Transaction* txn, const std::string& instance_id, const TabletIndexPB& idx, - TabletStatsPB& stats, bool snapshot) { + TabletStatsPB& tablet_stats, bool snapshot) { TabletStats detached_stats; - internal_get_tablet_stats(code, msg, txn, instance_id, idx, stats, detached_stats, snapshot); - merge_tablet_stats(stats, detached_stats); + internal_get_tablet_stats(code, msg, txn, instance_id, idx, tablet_stats, detached_stats, + snapshot); + merge_tablet_stats(tablet_stats, detached_stats); } MetaServiceResponseStatus parse_fix_tablet_stats_param( diff --git a/cloud/src/meta-service/meta_service_tablet_stats.h b/cloud/src/meta-service/meta_service_tablet_stats.h index 30cbaf56f6dc80..61d1cdc8e0c7b3 100644 --- a/cloud/src/meta-service/meta_service_tablet_stats.h +++ b/cloud/src/meta-service/meta_service_tablet_stats.h @@ -19,6 +19,7 @@ #include +#include "common/stats.h" #include "resource-manager/resource_manager.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 04d12657ed5d8d..f42ddfc5d8effa 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -110,7 +110,6 @@ void MetaServiceImpl::begin_txn(::google::protobuf::RpcController* controller, RPC_RATE_LIMIT(begin_txn) //1. Generate version stamp for txn id - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -379,7 +378,6 @@ void MetaServiceImpl::precommit_txn(::google::protobuf::RpcController* controlle return; } RPC_RATE_LIMIT(precommit_txn); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -609,7 +607,6 @@ void MetaServiceImpl::get_rl_task_commit_attach(::google::protobuf::RpcControlle } RPC_RATE_LIMIT(get_rl_task_commit_attach) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -678,7 +675,6 @@ void MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr } RPC_RATE_LIMIT(reset_rl_progress) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2846,7 +2842,6 @@ void MetaServiceImpl::abort_txn(::google::protobuf::RpcController* controller, } RPC_RATE_LIMIT(abort_txn); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2893,7 +2888,6 @@ void MetaServiceImpl::get_txn(::google::protobuf::RpcController* controller, } RPC_RATE_LIMIT(get_txn) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3010,7 +3004,7 @@ void MetaServiceImpl::get_current_max_txn_id(::google::protobuf::RpcController* return; } RPC_RATE_LIMIT(get_current_max_txn_id) - std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { msg = "failed to create txn"; @@ -3086,7 +3080,6 @@ void MetaServiceImpl::begin_sub_txn(::google::protobuf::RpcController* controlle } RPC_RATE_LIMIT(begin_sub_txn) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3284,7 +3277,6 @@ void MetaServiceImpl::abort_sub_txn(::google::protobuf::RpcController* controlle } RPC_RATE_LIMIT(abort_sub_txn) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3391,7 +3383,6 @@ void MetaServiceImpl::abort_txn_with_coordinator(::google::protobuf::RpcControll std::string end_info_key = txn_info_key({instance_id, INT64_MAX, INT64_MAX}); LOG(INFO) << "begin_info_key:" << hex(begin_info_key) << " end_info_key:" << hex(end_info_key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { msg = "failed to create txn"; @@ -3507,7 +3498,6 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont LOG(INFO) << "begin_running_key:" << hex(begin_running_key) << " end_running_key:" << hex(end_running_key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { msg = "failed to create txn"; @@ -3864,7 +3854,7 @@ void MetaServiceImpl::get_txn_id(::google::protobuf::RpcController* controller, const int64_t db_id = request->db_id(); std::string label = request->label(); const std::string label_key = txn_label_key({instance_id, db_id, label}); - std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to create txn. err=" << err << " db_id=" << db_id diff --git a/cloud/src/meta-service/txn_kv.cpp b/cloud/src/meta-service/txn_kv.cpp index a328538ed78832..2105c8d7ea1a67 100644 --- a/cloud/src/meta-service/txn_kv.cpp +++ b/cloud/src/meta-service/txn_kv.cpp @@ -407,6 +407,7 @@ TxnErrorCode Transaction::get(std::string_view key, std::string* val, bool snaps } if (!found) return TxnErrorCode::TXN_KEY_NOT_FOUND; + num_get_keys_++; *val = std::string((char*)ret, len); return TxnErrorCode::TXN_OK; } @@ -436,6 +437,7 @@ TxnErrorCode Transaction::get(std::string_view begin, std::string_view end, std::unique_ptr ret(new RangeGetIterator(fut)); RETURN_IF_ERROR(ret->init()); + num_get_keys_ += ret->size(); g_bvar_txn_kv_get_count_normalized << ret->size(); *(iter) = std::move(ret); @@ -666,6 +668,7 @@ TxnErrorCode Transaction::batch_get(std::vector>* res futures.clear(); } DCHECK_EQ(res->size(), num_keys); + num_get_keys_ += num_keys; return TxnErrorCode::TXN_OK; } diff --git a/cloud/src/meta-service/txn_kv.h b/cloud/src/meta-service/txn_kv.h index 645982f76164ef..66b01c37752b8c 100644 --- a/cloud/src/meta-service/txn_kv.h +++ b/cloud/src/meta-service/txn_kv.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -233,6 +234,11 @@ class Transaction { **/ virtual size_t approximate_bytes() const = 0; + /** + * @brief return the num get keys submitted to this txn. + **/ + virtual size_t num_get_keys() const = 0; + /** * @brief return the num delete keys submitted to this txn. **/ @@ -565,6 +571,8 @@ class Transaction : public cloud::Transaction { size_t approximate_bytes() const override { return approximate_bytes_; } + size_t num_get_keys() const override { return num_get_keys_; } + size_t num_del_keys() const override { return num_del_keys_; } size_t num_put_keys() const override { return num_put_keys_; } @@ -579,6 +587,7 @@ class Transaction : public cloud::Transaction { bool aborted_ = false; FDBTransaction* txn_ = nullptr; + size_t num_get_keys_ {0}; size_t num_del_keys_ {0}; size_t num_put_keys_ {0}; size_t delete_bytes_ {0}; diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp new file mode 100644 index 00000000000000..a8c6e4bdb09002 --- /dev/null +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -0,0 +1,949 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//#define private public +#include "common/bvars.h" +#include "meta-service/meta_service.h" +//#undef private + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "common/util.h" +#include "cpp/sync_point.h" +#include "meta-service/keys.h" +#include "meta-service/mem_txn_kv.h" +#include "meta-service/meta_service_helper.h" +#include "meta-service/txn_kv.h" +#include "meta-service/txn_kv_error.h" +#include "mock_resource_manager.h" +#include "rate-limiter/rate_limiter.h" +#include "resource-manager/resource_manager.h" + +int main(int argc, char** argv) { + const std::string conf_file = "/mnt/disk2/lihao/doris/cloud/conf/doris_cloud.conf"; + if (!doris::cloud::config::init(conf_file.c_str(), true)) { + std::cerr << "failed to init config file, conf=" << conf_file << std::endl; + return -1; + } + + if (!doris::cloud::init_glog("rpc_kv_bvar_test")) { + std::cerr << "failed to init glog" << std::endl; + return -1; + } + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +namespace doris::cloud { +using Status = MetaServiceResponseStatus; + +void start_compaction_job(MetaService* meta_service, int64_t tablet_id, const std::string& job_id, + const std::string& initiator, int base_compaction_cnt, + int cumu_compaction_cnt, TabletCompactionJobPB::CompactionType type, + StartTabletJobResponse& res, + std::pair input_version = {0, 0}) { + brpc::Controller cntl; + StartTabletJobRequest req; + req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id); + auto* compaction = req.mutable_job()->add_compaction(); + compaction->set_id(job_id); + compaction->set_initiator(initiator); + compaction->set_base_compaction_cnt(base_compaction_cnt); + compaction->set_cumulative_compaction_cnt(cumu_compaction_cnt); + compaction->set_type(type); + long now = time(nullptr); + compaction->set_expiration(now + 12); + compaction->set_lease(now + 3); + if (input_version.second > 0) { + compaction->add_input_versions(input_version.first); + compaction->add_input_versions(input_version.second); + compaction->set_check_input_versions_range(true); + } + meta_service->start_tablet_job(&cntl, &req, &res, nullptr); +}; + +std::unique_ptr get_meta_service(bool mock_resource_mgr) { + int ret = 0; + // MemKv + auto txn_kv = std::dynamic_pointer_cast(std::make_shared()); + if (txn_kv != nullptr) { + ret = txn_kv->init(); + [&] { ASSERT_EQ(ret, 0); }(); + } + [&] { ASSERT_NE(txn_kv.get(), nullptr); }(); + + std::unique_ptr txn; + EXPECT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->remove("\x00", "\xfe"); // This is dangerous if the fdb is not correctly set + EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + auto rs = std::make_shared(txn_kv); + auto rl = std::make_shared(); + auto meta_service = std::make_unique(txn_kv, rs, rl); + return std::make_unique(std::move(meta_service)); +} + +std::unique_ptr get_meta_service() { + return get_meta_service(true); +} + +std::unique_ptr get_fdb_meta_service() { + config::fdb_cluster_file_path = "fdb.cluster"; + static auto txn_kv = std::dynamic_pointer_cast(std::make_shared()); + static std::atomic init {false}; + bool tmp = false; + if (init.compare_exchange_strong(tmp, true)) { + int ret = txn_kv->init(); + [&] { + ASSERT_EQ(ret, 0); + ASSERT_NE(txn_kv.get(), nullptr); + }(); + } + auto rs = std::make_shared(txn_kv); + auto rl = std::make_shared(); + auto meta_service = std::make_unique(txn_kv, rs, rl); + return std::make_unique(std::move(meta_service)); +} + +static void commit_rowset(MetaServiceProxy* meta_service, const doris::RowsetMetaCloudPB& rowset, + CreateRowsetResponse& res) { + brpc::Controller cntl; + auto* arena = res.GetArena(); + auto* req = google::protobuf::Arena::CreateMessage(arena); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->commit_rowset(&cntl, req, &res, nullptr); + if (!arena) { + delete req; + } +} + +static std::string next_rowset_id() { + static int cnt = 0; + return std::to_string(++cnt); +} + +static void fill_schema(doris::TabletSchemaCloudPB* schema, int32_t schema_version) { + schema->set_schema_version(schema_version); + for (int i = 0; i < 10; ++i) { + auto* column = schema->add_column(); + column->set_unique_id(20000 + i); + column->set_type("INT"); + } +} + +static void add_tablet(CreateTabletsRequest& req, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, const std::string& rowset_id, + int32_t schema_version) { + auto* tablet = req.add_tablet_metas(); + tablet->set_table_id(table_id); + tablet->set_index_id(index_id); + tablet->set_partition_id(partition_id); + tablet->set_tablet_id(tablet_id); + auto* schema = tablet->mutable_schema(); + fill_schema(schema, schema_version); + auto* first_rowset = tablet->add_rs_metas(); + first_rowset->set_rowset_id(0); // required + first_rowset->set_rowset_id_v2(rowset_id); + first_rowset->set_start_version(0); + first_rowset->set_end_version(1); + first_rowset->mutable_tablet_schema()->CopyFrom(*schema); +} + +static void add_tablet(CreateTabletsRequest& req, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id) { + auto* tablet = req.add_tablet_metas(); + tablet->set_table_id(table_id); + tablet->set_index_id(index_id); + tablet->set_partition_id(partition_id); + tablet->set_tablet_id(tablet_id); + auto* schema = tablet->mutable_schema(); + schema->set_schema_version(0); + auto* first_rowset = tablet->add_rs_metas(); + first_rowset->set_rowset_id(0); // required + first_rowset->set_rowset_id_v2(next_rowset_id()); + first_rowset->set_start_version(0); + first_rowset->set_end_version(1); + first_rowset->mutable_tablet_schema()->CopyFrom(*schema); +} + +static void create_tablet_with_db_id(MetaServiceProxy* meta_service, int64_t db_id, + int64_t table_id, int64_t index_id, int64_t partition_id, + int64_t tablet_id) { + brpc::Controller cntl; + CreateTabletsRequest req; + CreateTabletsResponse res; + req.set_db_id(db_id); + add_tablet(req, table_id, index_id, partition_id, tablet_id); + meta_service->create_tablets(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; +} + +static void create_tablet(MetaServiceProxy* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, const std::string& rowset_id, + int32_t schema_version) { + brpc::Controller cntl; + CreateTabletsRequest req; + CreateTabletsResponse res; + add_tablet(req, table_id, index_id, partition_id, tablet_id, rowset_id, schema_version); + meta_service->create_tablets(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; +} + +static void create_tablet(MetaServiceProxy* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id) { + brpc::Controller cntl; + CreateTabletsRequest req; + CreateTabletsResponse res; + add_tablet(req, table_id, index_id, partition_id, tablet_id); + meta_service->create_tablets(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; +} + +static void begin_txn(MetaServiceProxy* meta_service, int64_t db_id, const std::string& label, + int64_t table_id, int64_t& txn_id) { + brpc::Controller cntl; + BeginTxnRequest req; + BeginTxnResponse res; + auto* txn_info = req.mutable_txn_info(); + txn_info->set_db_id(db_id); + txn_info->set_label(label); + txn_info->add_table_ids(table_id); + txn_info->set_timeout_ms(36000); + meta_service->begin_txn(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label; + ASSERT_TRUE(res.has_txn_id()) << label; + txn_id = res.txn_id(); +} + +static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id, + int partition_id = 10, int64_t version = -1, + int num_rows = 100) { + doris::RowsetMetaCloudPB rowset; + rowset.set_rowset_id(0); // required + rowset.set_rowset_id_v2(next_rowset_id()); + rowset.set_tablet_id(tablet_id); + rowset.set_partition_id(partition_id); + rowset.set_txn_id(txn_id); + if (version > 0) { + rowset.set_start_version(version); + rowset.set_end_version(version); + } + rowset.set_num_segments(1); + rowset.set_num_rows(num_rows); + rowset.set_data_disk_size(num_rows * 100); + rowset.set_index_disk_size(num_rows * 10); + rowset.set_total_disk_size(num_rows * 110); + rowset.mutable_tablet_schema()->set_schema_version(0); + rowset.set_txn_expiration(::time(nullptr)); // Required by DCHECK + return rowset; +} + +static void check_get_tablet(MetaServiceProxy* meta_service, int64_t tablet_id, + int32_t schema_version) { + brpc::Controller cntl; + GetTabletRequest req; + GetTabletResponse res; + req.set_tablet_id(tablet_id); + meta_service->get_tablet(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; + ASSERT_TRUE(res.has_tablet_meta()) << tablet_id; + EXPECT_TRUE(res.tablet_meta().has_schema()) << tablet_id; + EXPECT_EQ(res.tablet_meta().schema_version(), schema_version) << tablet_id; +}; + +static void prepare_rowset(MetaServiceProxy* meta_service, const doris::RowsetMetaCloudPB& rowset, + CreateRowsetResponse& res) { + brpc::Controller cntl; + auto* arena = res.GetArena(); + auto* req = google::protobuf::Arena::CreateMessage(arena); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->prepare_rowset(&cntl, req, &res, nullptr); + if (!arena) { + delete req; + } +} + +static void get_tablet_stats(MetaService* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, GetTabletStatsResponse& res) { + brpc::Controller cntl; + GetTabletStatsRequest req; + auto* idx = req.add_tablet_idx(); + idx->set_table_id(table_id); + idx->set_index_id(index_id); + idx->set_partition_id(partition_id); + idx->set_tablet_id(tablet_id); + meta_service->get_tablet_stats(&cntl, &req, &res, nullptr); +} + +static void commit_txn(MetaServiceProxy* meta_service, int64_t db_id, int64_t txn_id, + const std::string& label) { + brpc::Controller cntl; + CommitTxnRequest req; + CommitTxnResponse res; + req.set_db_id(db_id); + req.set_txn_id(txn_id); + meta_service->commit_txn(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label; +} + +static void begin_txn_and_commit_rowset(MetaServiceProxy* meta_service, const std::string& label, + int64_t db_id, int64_t table_id, int64_t partition_id, + int64_t tablet_id, int64_t* txn_id) { + begin_txn(meta_service, db_id, label, table_id, *txn_id); + CreateRowsetResponse res; + auto rowset = create_rowset(*txn_id, tablet_id, partition_id); + prepare_rowset(meta_service, rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + res.Clear(); + commit_rowset(meta_service, rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); +} + +static void insert_rowset(MetaServiceProxy* meta_service, int64_t db_id, const std::string& label, + int64_t table_id, int64_t partition_id, int64_t tablet_id) { + int64_t txn_id = 0; + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service, db_id, label, table_id, txn_id)); + CreateRowsetResponse res; + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + prepare_rowset(meta_service, rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label; + res.Clear(); + ASSERT_NO_FATAL_FAILURE(commit_rowset(meta_service, rowset, res)); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label; + commit_txn(meta_service, db_id, txn_id, label); +} + +static void get_rowset(MetaServiceProxy* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, GetRowsetResponse& res) { + brpc::Controller cntl; + GetRowsetRequest req; + auto* tablet_idx = req.mutable_idx(); + tablet_idx->set_table_id(table_id); + tablet_idx->set_index_id(index_id); + tablet_idx->set_partition_id(partition_id); + tablet_idx->set_tablet_id(tablet_id); + req.set_start_version(0); + req.set_end_version(-1); + req.set_cumulative_compaction_cnt(0); + req.set_base_compaction_cnt(0); + req.set_cumulative_point(2); + meta_service->get_rowset(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; +} + +static void get_delete_bitmap_update_lock(MetaServiceProxy* meta_service, int64_t table_id, + int64_t partition_id, int64_t lock_id, + int64_t initiator) { + brpc::Controller cntl; + GetDeleteBitmapUpdateLockRequest get_lock_req; + GetDeleteBitmapUpdateLockResponse get_lock_res; + get_lock_req.set_cloud_unique_id("test_cloud_unique_id"); + get_lock_req.set_table_id(table_id); + get_lock_req.add_partition_ids(partition_id); + get_lock_req.set_expiration(5); + get_lock_req.set_lock_id(lock_id); + get_lock_req.set_initiator(initiator); + meta_service->get_delete_bitmap_update_lock( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &get_lock_req, + &get_lock_res, nullptr); + ASSERT_EQ(get_lock_res.status().code(), MetaServiceCode::OK); +} + +static void update_delete_bitmap(MetaServiceProxy* meta_service, + UpdateDeleteBitmapRequest& update_delete_bitmap_req, + UpdateDeleteBitmapResponse& update_delete_bitmap_res, + int64_t table_id, int64_t partition_id, int64_t lock_id, + int64_t initiator, int64_t tablet_id, int64_t txn_id, + int64_t next_visible_version, std::string data = "1111") { + brpc::Controller cntl; + update_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + update_delete_bitmap_req.set_table_id(table_id); + update_delete_bitmap_req.set_partition_id(partition_id); + update_delete_bitmap_req.set_lock_id(lock_id); + update_delete_bitmap_req.set_initiator(initiator); + update_delete_bitmap_req.set_tablet_id(tablet_id); + update_delete_bitmap_req.set_txn_id(txn_id); + update_delete_bitmap_req.set_next_visible_version(next_visible_version); + update_delete_bitmap_req.add_rowset_ids("123"); + update_delete_bitmap_req.add_segment_ids(0); + update_delete_bitmap_req.add_versions(next_visible_version); + update_delete_bitmap_req.add_segment_delete_bitmaps(data); + meta_service->update_delete_bitmap(reinterpret_cast(&cntl), + &update_delete_bitmap_req, &update_delete_bitmap_res, + nullptr); +} + +// create_tablets +TEST(RpcKvBvarTest, DISABLED_CreateTablets) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + LOG(INFO) << "CreateTablets: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_create_tablets_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_create_tablets_write_counter.get_value()); +} + +// get_tablet +TEST(RpcKvBvarTest, DISABLED_GetTablet) { + std::string cloud_unique_id = "test_cloud_unique_id"; + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + brpc::Controller cntl; + GetTabletRequest req; + req.set_cloud_unique_id(cloud_unique_id); + req.set_tablet_id(tablet_id); + GetTabletResponse resp; + + meta_service->get_tablet(&cntl, &req, &resp, nullptr); + LOG(INFO) << "GetTablet: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_tablet_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_tablet_write_counter.get_value()); +} + +// get_tablet_stats +TEST(RpcKvBvarTest, DISABLED_GetTabletStats) { + std::string cloud_unique_id = "test_cloud_unique_id"; + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + GetTabletStatsResponse res; + get_tablet_stats(meta_service.get(), table_id, index_id, partition_id, tablet_id, res); + + LOG(INFO) << "GetTabletStats: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_tablet_stats_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_tablet_stats_write_counter.get_value()); +} + +// update_tablet +TEST(RpcKvBvarTest, DISABLED_UpdateTablet) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + brpc::Controller cntl; + UpdateTabletRequest req; + UpdateTabletResponse resp; + req.set_cloud_unique_id("test_cloud_unique_id"); + TabletMetaInfoPB* tablet_meta_info = req.add_tablet_meta_infos(); + tablet_meta_info->set_tablet_id(tablet_id); + tablet_meta_info->set_is_in_memory(true); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + meta_service->update_tablet(&cntl, &req, &resp, nullptr); + + LOG(INFO) << "UpdateTablet: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_update_tablet_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_update_tablet_write_counter.get_value()); +} + +// update_tablet_schema +// should not call update_tablet_schema +// TEST(RpcKvBvarTest, DISABLED_UpdateTabletSchema) { +// auto meta_service = get_meta_service(); +// auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); +// constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; +// create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + +// brpc::Controller cntl; +// UpdateTabletSchemaRequest req; +// UpdateTabletSchemaResponse resp; +// req.set_tablet_id(tablet_id); +// req.set_cloud_unique_id("test_cloud_unique_id"); + +// mem_kv->read_count_ = 0; +// mem_kv->write_count_ = 0; + +// meta_service->update_tablet_schema(&cntl, &req, &resp, nullptr); + +// LOG(INFO) << "UpdateTabletSchema: " << mem_kv->read_count_ << ", " +// << mem_kv->write_count_; +// ASSERT_EQ(mem_kv->read_count_, +// g_bvar_rpc_kv_update_tablet_schema_read_counter.get_value()); +// ASSERT_EQ(mem_kv->write_count_, +// g_bvar_rpc_kv_update_tablet_schema_write_counter.get_value()); +// } + +// prepare_rowset +TEST(RpcKvBvarTest, DISABLED_PrepareRowset) { + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + int64_t txn_id = 0; + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + CreateRowsetResponse res; + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + rowset.mutable_load_id()->set_hi(123); + rowset.mutable_load_id()->set_lo(456); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + prepare_rowset(meta_service.get(), rowset, res); + + LOG(INFO) << "PrepareRowset: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_prepare_rowset_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_prepare_rowset_write_counter.get_value()); +} + +// get_rowset +TEST(RpcKvBvarTest, DISABLED_GetRowset) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; + config::write_schema_kv = true; + ASSERT_NO_FATAL_FAILURE(create_tablet(meta_service.get(), table_id, index_id, partition_id, + tablet_id, next_rowset_id(), 1)); + // check get tablet response + check_get_tablet(meta_service.get(), tablet_id, 1); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + // check get rowset response + GetRowsetResponse get_rowset_res; + get_rowset(meta_service.get(), table_id, index_id, partition_id, tablet_id, get_rowset_res); + + LOG(INFO) << "GetRowset: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_rowset_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_rowset_write_counter.get_value()); +} + +// update_tmp_rowset +TEST(RpcKvBvarTest, DISABLED_UpdateTmpRowset) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; + int64_t txn_id = -1; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + rowset.set_num_segments(rowset.num_segments() + 3); + rowset.set_num_rows(rowset.num_rows() + 1000); + rowset.set_total_disk_size(rowset.total_disk_size() + 11000); + rowset.set_index_disk_size(rowset.index_disk_size() + 1000); + rowset.set_data_disk_size(rowset.data_disk_size() + 10000); + + std::unique_ptr txn; + std::string update_key; + brpc::Controller cntl; + CreateRowsetResponse res; + auto* arena = res.GetArena(); + auto* req = google::protobuf::Arena::CreateMessage(arena); + std::string instance_id = get_instance_id(meta_service->resource_mgr(), req->cloud_unique_id()); + MetaRowsetTmpKeyInfo key_info {instance_id, txn_id, tablet_id}; + meta_rowset_tmp_key(key_info, &update_key); + EXPECT_EQ(mem_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(update_key, "update_tmp_rowset_val"); + EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + req->mutable_rowset_meta()->CopyFrom(rowset); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + meta_service->update_tmp_rowset(&cntl, req, &res, nullptr); + + if (!arena) { + delete req; + } + + LOG(INFO) << "UpdateTmpRowset: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_update_tmp_rowset_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_update_tmp_rowset_write_counter.get_value()); +} + +// commit_rowset +TEST(RpcKvBvarTest, DISABLED_CommitRowset) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; + int64_t txn_id = -1; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + auto tmp_rowset = create_rowset(txn_id, tablet_id, partition_id); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + + LOG(INFO) << "CommitRowset: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_commit_rowset_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_commit_rowset_write_counter.get_value()); +} + +// get_version +TEST(RpcKvBvarTest, DISABLED_GetVersion) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + constexpr auto table_id = 10001, partition_id = 10003, tablet_id = 10004; + create_tablet(meta_service.get(), table_id, 1, partition_id, tablet_id); + insert_rowset(meta_service.get(), 1, "get_version_label_1", table_id, partition_id, tablet_id); + + brpc::Controller ctrl; + GetVersionRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_db_id(1); + req.set_table_id(table_id); + req.set_partition_id(partition_id); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + GetVersionResponse resp; + meta_service->get_version(&ctrl, &req, &resp, nullptr); + + LOG(INFO) << "GetVersion: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_version_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_version_write_counter.get_value()); +} + +// get_schema_dict +TEST(RpcKvBvarTest, DISABLED_GetSchemaDict) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + brpc::Controller ctrl; + GetSchemaDictRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_index_id(index_id); + + std::unique_ptr txn; + std::string instance_id = get_instance_id(meta_service->resource_mgr(), req.cloud_unique_id()); + std::string dict_key = meta_schema_pb_dictionary_key({instance_id, req.index_id()}); + EXPECT_EQ(mem_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(dict_key, "dict_val"); + EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + GetSchemaDictResponse resp; + meta_service->get_schema_dict(&ctrl, &req, &resp, nullptr); + + LOG(INFO) << "GetSchemaDict: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_schema_dict_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_schema_dict_write_counter.get_value()); +} + +// get_delete_bitmap_update_lock +TEST(RpcKvBvarTest, DISABLED_GetDeleteBitmapUpdateLock) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label11", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + + LOG(INFO) << "GetDeleteBitmapUpdateLock: " << mem_kv->read_count_ << ", " + << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_write_counter.get_value()); +} + +// update_delete_bitmap +TEST(RpcKvBvarTest, DISABLED_UpdateDeleteBitmap) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + size_t split_size = 90 * 1000; // see cloud/src/common/util.h + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label11", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + int64_t version = 100; + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + // will be splited and stored in 5 KVs + std::string data1(split_size * 5, 'c'); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + update_delete_bitmap(meta_service.get(), update_delete_bitmap_req, update_delete_bitmap_res, + table_id, t1p1, lock_id, initiator, tablet_id, txn_id, version, data1); + + LOG(INFO) << "UpdateDeleteBitmap: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_update_delete_bitmap_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_update_delete_bitmap_write_counter.get_value()); +} + +// get_delete_bitmap +TEST(RpcKvBvarTest, DISABLED_GetDeleteBitmap) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + size_t split_size = 90 * 1000; // see cloud/src/common/util.h + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label11", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + int64_t version = 100; + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + // will be splited and stored in 5 KVs + std::string data1(split_size * 5, 'c'); + update_delete_bitmap(meta_service.get(), update_delete_bitmap_req, update_delete_bitmap_res, + table_id, t1p1, lock_id, initiator, tablet_id, txn_id, version, data1); + + brpc::Controller ctrl; + GetDeleteBitmapRequest get_delete_bitmap_req; + GetDeleteBitmapResponse get_delete_bitmap_res; + get_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + get_delete_bitmap_req.set_tablet_id(tablet_id); + get_delete_bitmap_req.add_rowset_ids("123"); + get_delete_bitmap_req.add_begin_versions(0); + get_delete_bitmap_req.add_end_versions(version); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + meta_service->get_delete_bitmap(reinterpret_cast(&ctrl), + &get_delete_bitmap_req, &get_delete_bitmap_res, nullptr); + + LOG(INFO) << "GetDeleteBitmap: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_delete_bitmap_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_delete_bitmap_write_counter.get_value()); +} + +// remove_delete_bitmap_update_lock +TEST(RpcKvBvarTest, DISABLED_RemoveDeleteBitmapUpdateLock) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label11", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + brpc::Controller cntl; + RemoveDeleteBitmapUpdateLockRequest remove_req; + RemoveDeleteBitmapUpdateLockResponse remove_res; + + remove_req.set_cloud_unique_id("test_cloud_unique_id"); + remove_req.set_table_id(table_id); + remove_req.set_lock_id(lock_id); + remove_req.set_initiator(initiator); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + meta_service->remove_delete_bitmap_update_lock( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &remove_req, &remove_res, + nullptr); + + LOG(INFO) << "RemoveDeleteBitmapUpdateLock: " << mem_kv->read_count_ << ", " + << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_write_counter.get_value()); +} + +// remove_delete_bitmap +TEST(RpcKvBvarTest, DISABLED_RemoveDeleteBitmap) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + size_t split_size = 90 * 1000; // see cloud/src/common/util.h + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label1", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + int64_t version = 100; + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + // will be splited and stored in 5 KVs + std::string data1(split_size * 5, 'c'); + update_delete_bitmap(meta_service.get(), update_delete_bitmap_req, update_delete_bitmap_res, + table_id, t1p1, lock_id, initiator, tablet_id, txn_id, version, data1); + + brpc::Controller ctrl; + RemoveDeleteBitmapRequest req; + RemoveDeleteBitmapResponse resp; + req.add_begin_versions(version); + req.add_end_versions(version); + req.add_rowset_ids("rowset_ids"); + req.set_cloud_unique_id("test_cloud_unique_id"); + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + meta_service->remove_delete_bitmap(&ctrl, &req, &resp, nullptr); + + LOG(INFO) << "RemoveDeleteBitmap: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_remove_delete_bitmap_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_remove_delete_bitmap_write_counter.get_value()); +} + +// start_tablet_job +TEST(RpcKvBvarTest, DISABLED_StartTabletJob) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + constexpr int64_t tablet_id = 10004; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + StartTabletJobResponse res; + + mem_kv->read_count_ = 0; + mem_kv->write_count_ = 0; + + start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, + TabletCompactionJobPB::BASE, res); + + LOG(INFO) << "StartTabletJob: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; + ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_start_tablet_job_read_counter.get_value()); + ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_start_tablet_job_write_counter.get_value()); +} + +// finish_tablet_job + +// prepare_index +// commit_index +// drop_index +// prepare_partition +// commit_partition +// drop_partition +// check_kv +// get_obj_store_info +// alter_storage_vault +// alter_obj_store_info +// update_ak_sk +// create_instance +// get_instance +// alter_cluster +// get_cluster +// create_stage +// get_stage +// get_iam +// alter_iam +// alter_ram_user +// begin_copy +// finish_copy +// get_copy_job +// get_copy_files +// filter_copy_files +// get_cluster_status +// begin_txn +// precommit_txn +// get_rl_task_commit_attach +// reset_rl_progress +// commit_txn +// abort_txn +// get_txn +// get_current_max_txn_id +// begin_sub_txn +// abort_sub_txn +// abort_txn_with_coordinator +// check_txn_conflict +// clean_txn_label +// get_txn_id + +} // namespace doris::cloud From 490aac0bb79918baa2fa8f7cf53bd77635780702 Mon Sep 17 00:00:00 2001 From: koarz Date: Tue, 17 Jun 2025 17:38:44 +0800 Subject: [PATCH 02/23] update --- cloud/src/common/bvars.cpp | 290 +- cloud/src/common/bvars.h | 290 +- cloud/src/common/stats.h | 10 +- cloud/src/meta-service/mem_txn_kv.cpp | 17 +- cloud/src/meta-service/mem_txn_kv.h | 6 +- cloud/src/meta-service/meta_service.cpp | 83 +- cloud/src/meta-service/meta_service.h | 10 +- cloud/src/meta-service/meta_service_helper.h | 17 +- .../meta-service/meta_service_partition.cpp | 12 +- .../meta-service/meta_service_resource.cpp | 29 +- .../meta_service_tablet_stats.cpp | 8 +- .../meta-service/meta_service_tablet_stats.h | 1 - cloud/src/meta-service/meta_service_txn.cpp | 62 +- cloud/src/meta-service/txn_lazy_committer.cpp | 6 +- cloud/test/meta_server_test.cpp | 6 +- cloud/test/rpc_kv_bvar_test.cpp | 2503 ++++++++++++++++- 16 files changed, 2913 insertions(+), 437 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index abedc38e77c58a..4b691649eea9cb 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -190,178 +190,236 @@ BvarStatusWithTag g_bvar_max_rowsets_with_useless_delete_bitmap_version // rpc kv rw count // get_rowset -bvar::Adder g_bvar_rpc_kv_get_rowset_read_counter("rpc_kv","get_rowset_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_rowset_write_counter("rpc_kv","get_rowset_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_rowset_get_counter("rpc_kv","get_rowset_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_rowset_put_counter("rpc_kv","get_rowset_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_rowset_del_counter("rpc_kv","get_rowset_del_counter"); // get_version -bvar::Adder g_bvar_rpc_kv_get_version_read_counter("rpc_kv","get_version_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_version_write_counter("rpc_kv","get_version_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_version_get_counter("rpc_kv","get_version_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_version_put_counter("rpc_kv","get_version_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_version_del_counter("rpc_kv","get_version_del_counter"); // get_schema_dict -bvar::Adder g_bvar_rpc_kv_get_schema_dict_read_counter("rpc_kv","get_schema_dict_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_schema_dict_write_counter("rpc_kv","get_schema_dict_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_schema_dict_get_counter("rpc_kv","get_schema_dict_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_schema_dict_put_counter("rpc_kv","get_schema_dict_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_schema_dict_del_counter("rpc_kv","get_schema_dict_del_counter"); // create_tablets -bvar::Adder g_bvar_rpc_kv_create_tablets_read_counter("rpc_kv","create_tablets_read_counter"); -bvar::Adder g_bvar_rpc_kv_create_tablets_write_counter("rpc_kv","create_tablets_write_counter"); +bvar::Adder g_bvar_rpc_kv_create_tablets_get_counter("rpc_kv","create_tablets_get_counter"); +bvar::Adder g_bvar_rpc_kv_create_tablets_put_counter("rpc_kv","create_tablets_put_counter"); +bvar::Adder g_bvar_rpc_kv_create_tablets_del_counter("rpc_kv","create_tablets_del_counter"); // update_tablet -bvar::Adder g_bvar_rpc_kv_update_tablet_read_counter("rpc_kv","update_tablet_read_counter"); -bvar::Adder g_bvar_rpc_kv_update_tablet_write_counter("rpc_kv","update_tablet_write_counter"); +bvar::Adder g_bvar_rpc_kv_update_tablet_get_counter("rpc_kv","update_tablet_get_counter"); +bvar::Adder g_bvar_rpc_kv_update_tablet_put_counter("rpc_kv","update_tablet_put_counter"); +bvar::Adder g_bvar_rpc_kv_update_tablet_del_counter("rpc_kv","update_tablet_del_counter"); // update_tablet_schema -bvar::Adder g_bvar_rpc_kv_update_tablet_schema_read_counter("rpc_kv","update_tablet_schema_read_counter"); -bvar::Adder g_bvar_rpc_kv_update_tablet_schema_write_counter("rpc_kv","update_tablet_schema_write_counter"); +bvar::Adder g_bvar_rpc_kv_update_tablet_schema_get_counter("rpc_kv","update_tablet_schema_get_counter"); +bvar::Adder g_bvar_rpc_kv_update_tablet_schema_put_counter("rpc_kv","update_tablet_schema_put_counter"); +bvar::Adder g_bvar_rpc_kv_update_tablet_schema_del_counter("rpc_kv","update_tablet_schema_del_counter"); // get_tablet -bvar::Adder g_bvar_rpc_kv_get_tablet_read_counter("rpc_kv","get_tablet_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_tablet_write_counter("rpc_kv","get_tablet_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_tablet_get_counter("rpc_kv","get_tablet_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_tablet_put_counter("rpc_kv","get_tablet_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_tablet_del_counter("rpc_kv","get_tablet_del_counter"); // prepare_rowset -bvar::Adder g_bvar_rpc_kv_prepare_rowset_read_counter("rpc_kv","prepare_rowset_read_counter"); -bvar::Adder g_bvar_rpc_kv_prepare_rowset_write_counter("rpc_kv","prepare_rowset_write_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_rowset_get_counter("rpc_kv","prepare_rowset_get_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_rowset_put_counter("rpc_kv","prepare_rowset_put_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_rowset_del_counter("rpc_kv","prepare_rowset_del_counter"); // commit_rowset -bvar::Adder g_bvar_rpc_kv_commit_rowset_read_counter("rpc_kv","commit_rowset_read_counter"); -bvar::Adder g_bvar_rpc_kv_commit_rowset_write_counter("rpc_kv","commit_rowset_write_counter"); +bvar::Adder g_bvar_rpc_kv_commit_rowset_get_counter("rpc_kv","commit_rowset_get_counter"); +bvar::Adder g_bvar_rpc_kv_commit_rowset_put_counter("rpc_kv","commit_rowset_put_counter"); +bvar::Adder g_bvar_rpc_kv_commit_rowset_del_counter("rpc_kv","commit_rowset_del_counter"); // update_tmp_rowset -bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_read_counter("rpc_kv","update_tmp_rowset_read_counter"); -bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_write_counter("rpc_kv","update_tmp_rowset_write_counter"); +bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_get_counter("rpc_kv","update_tmp_rowset_get_counter"); +bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_put_counter("rpc_kv","update_tmp_rowset_put_counter"); +bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_del_counter("rpc_kv","update_tmp_rowset_del_counter"); // get_tablet_stats -bvar::Adder g_bvar_rpc_kv_get_tablet_stats_read_counter("rpc_kv","get_tablet_stats_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_tablet_stats_write_counter("rpc_kv","get_tablet_stats_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_tablet_stats_get_counter("rpc_kv","get_tablet_stats_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_tablet_stats_put_counter("rpc_kv","get_tablet_stats_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_tablet_stats_del_counter("rpc_kv","get_tablet_stats_del_counter"); // update_delete_bitmap -bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_read_counter("rpc_kv","update_delete_bitmap_read_counter"); -bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_write_counter("rpc_kv","update_delete_bitmap_write_counter"); +bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_get_counter("rpc_kv","update_delete_bitmap_get_counter"); +bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_put_counter("rpc_kv","update_delete_bitmap_put_counter"); +bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_del_counter("rpc_kv","update_delete_bitmap_del_counter"); // get_delete_bitmap -bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_read_counter("rpc_kv","get_delete_bitmap_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_write_counter("rpc_kv","get_delete_bitmap_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_get_counter("rpc_kv","get_delete_bitmap_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_put_counter("rpc_kv","get_delete_bitmap_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_del_counter("rpc_kv","get_delete_bitmap_del_counter"); // get_delete_bitmap_update_lock -bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_read_counter("rpc_kv","get_delete_bitmap_update_lock_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_write_counter("rpc_kv","get_delete_bitmap_update_lock_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter("rpc_kv","get_delete_bitmap_update_lock_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter("rpc_kv","get_delete_bitmap_update_lock_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter("rpc_kv","get_delete_bitmap_update_lock_del_counter"); // remove_delete_bitmap_update_lock -bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_read_counter("rpc_kv","remove_delete_bitmap_update_lock_read_counter"); -bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_write_counter("rpc_kv","remove_delete_bitmap_update_lock_write_counter"); +bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter("rpc_kv","remove_delete_bitmap_update_lock_get_counter"); +bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter("rpc_kv","remove_delete_bitmap_update_lock_put_counter"); +bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter("rpc_kv","remove_delete_bitmap_update_lock_del_counter"); // remove_delete_bitmap -bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_read_counter("rpc_kv","remove_delete_bitmap_read_counter"); -bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_write_counter("rpc_kv","remove_delete_bitmap_write_counter"); +bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_get_counter("rpc_kv","remove_delete_bitmap_get_counter"); +bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_put_counter("rpc_kv","remove_delete_bitmap_put_counter"); +bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_del_counter("rpc_kv","remove_delete_bitmap_del_counter"); // start_tablet_job -bvar::Adder g_bvar_rpc_kv_start_tablet_job_read_counter("rpc_kv","start_tablet_job_read_counter"); -bvar::Adder g_bvar_rpc_kv_start_tablet_job_write_counter("rpc_kv","start_tablet_job_write_counter"); +bvar::Adder g_bvar_rpc_kv_start_tablet_job_get_counter("rpc_kv","start_tablet_job_get_counter"); +bvar::Adder g_bvar_rpc_kv_start_tablet_job_put_counter("rpc_kv","start_tablet_job_put_counter"); +bvar::Adder g_bvar_rpc_kv_start_tablet_job_del_counter("rpc_kv","start_tablet_job_del_counter"); // finish_tablet_job -bvar::Adder g_bvar_rpc_kv_finish_tablet_job_read_counter("rpc_kv","finish_tablet_job_read_counter"); -bvar::Adder g_bvar_rpc_kv_finish_tablet_job_write_counter("rpc_kv","finish_tablet_job_write_counter"); +bvar::Adder g_bvar_rpc_kv_finish_tablet_job_get_counter("rpc_kv","finish_tablet_job_get_counter"); +bvar::Adder g_bvar_rpc_kv_finish_tablet_job_put_counter("rpc_kv","finish_tablet_job_put_counter"); +bvar::Adder g_bvar_rpc_kv_finish_tablet_job_del_counter("rpc_kv","finish_tablet_job_del_counter"); // prepare_index -bvar::Adder g_bvar_rpc_kv_prepare_index_read_counter("rpc_kv","prepare_index_read_counter"); -bvar::Adder g_bvar_rpc_kv_prepare_index_write_counter("rpc_kv","prepare_index_write_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_index_get_counter("rpc_kv","prepare_index_get_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_index_put_counter("rpc_kv","prepare_index_put_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_index_del_counter("rpc_kv","prepare_index_del_counter"); // commit_index -bvar::Adder g_bvar_rpc_kv_commit_index_read_counter("rpc_kv","commit_index_read_counter"); -bvar::Adder g_bvar_rpc_kv_commit_index_write_counter("rpc_kv","commit_index_write_counter"); +bvar::Adder g_bvar_rpc_kv_commit_index_get_counter("rpc_kv","commit_index_get_counter"); +bvar::Adder g_bvar_rpc_kv_commit_index_put_counter("rpc_kv","commit_index_put_counter"); +bvar::Adder g_bvar_rpc_kv_commit_index_del_counter("rpc_kv","commit_index_del_counter"); // drop_index -bvar::Adder g_bvar_rpc_kv_drop_index_read_counter("rpc_kv","drop_index_read_counter"); -bvar::Adder g_bvar_rpc_kv_drop_index_write_counter("rpc_kv","drop_index_write_counter"); +bvar::Adder g_bvar_rpc_kv_drop_index_get_counter("rpc_kv","drop_index_get_counter"); +bvar::Adder g_bvar_rpc_kv_drop_index_put_counter("rpc_kv","drop_index_put_counter"); +bvar::Adder g_bvar_rpc_kv_drop_index_del_counter("rpc_kv","drop_index_del_counter"); // prepare_partition -bvar::Adder g_bvar_rpc_kv_prepare_partition_read_counter("rpc_kv","prepare_partition_read_counter"); -bvar::Adder g_bvar_rpc_kv_prepare_partition_write_counter("rpc_kv","prepare_partition_write_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_partition_get_counter("rpc_kv","prepare_partition_get_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_partition_put_counter("rpc_kv","prepare_partition_put_counter"); +bvar::Adder g_bvar_rpc_kv_prepare_partition_del_counter("rpc_kv","prepare_partition_del_counter"); // commit_partition -bvar::Adder g_bvar_rpc_kv_commit_partition_read_counter("rpc_kv","commit_partition_read_counter"); -bvar::Adder g_bvar_rpc_kv_commit_partition_write_counter("rpc_kv","commit_partition_write_counter"); +bvar::Adder g_bvar_rpc_kv_commit_partition_get_counter("rpc_kv","commit_partition_get_counter"); +bvar::Adder g_bvar_rpc_kv_commit_partition_put_counter("rpc_kv","commit_partition_put_counter"); +bvar::Adder g_bvar_rpc_kv_commit_partition_del_counter("rpc_kv","commit_partition_del_counter"); // drop_partition -bvar::Adder g_bvar_rpc_kv_drop_partition_read_counter("rpc_kv","drop_partition_read_counter"); -bvar::Adder g_bvar_rpc_kv_drop_partition_write_counter("rpc_kv","drop_partition_write_counter"); +bvar::Adder g_bvar_rpc_kv_drop_partition_get_counter("rpc_kv","drop_partition_get_counter"); +bvar::Adder g_bvar_rpc_kv_drop_partition_put_counter("rpc_kv","drop_partition_put_counter"); +bvar::Adder g_bvar_rpc_kv_drop_partition_del_counter("rpc_kv","drop_partition_del_counter"); // check_kv -bvar::Adder g_bvar_rpc_kv_check_kv_read_counter("rpc_kv","check_kv_read_counter"); -bvar::Adder g_bvar_rpc_kv_check_kv_write_counter("rpc_kv","check_kv_write_counter"); +bvar::Adder g_bvar_rpc_kv_check_kv_get_counter("rpc_kv","check_kv_get_counter"); +bvar::Adder g_bvar_rpc_kv_check_kv_put_counter("rpc_kv","check_kv_put_counter"); +bvar::Adder g_bvar_rpc_kv_check_kv_del_counter("rpc_kv","check_kv_del_counter"); // get_obj_store_info -bvar::Adder g_bvar_rpc_kv_get_obj_store_info_read_counter("rpc_kv","get_obj_store_info_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_obj_store_info_write_counter("rpc_kv","get_obj_store_info_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_obj_store_info_get_counter("rpc_kv","get_obj_store_info_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_obj_store_info_put_counter("rpc_kv","get_obj_store_info_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_obj_store_info_del_counter("rpc_kv","get_obj_store_info_del_counter"); // alter_storage_vault -bvar::Adder g_bvar_rpc_kv_alter_storage_vault_read_counter("rpc_kv","alter_storage_vault_read_counter"); -bvar::Adder g_bvar_rpc_kv_alter_storage_vault_write_counter("rpc_kv","alter_storage_vault_write_counter"); +bvar::Adder g_bvar_rpc_kv_alter_storage_vault_get_counter("rpc_kv","alter_storage_vault_get_counter"); +bvar::Adder g_bvar_rpc_kv_alter_storage_vault_put_counter("rpc_kv","alter_storage_vault_put_counter"); +bvar::Adder g_bvar_rpc_kv_alter_storage_vault_del_counter("rpc_kv","alter_storage_vault_del_counter"); // alter_obj_store_info -bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_read_counter("rpc_kv","alter_obj_store_info_read_counter"); -bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_write_counter("rpc_kv","alter_obj_store_info_write_counter"); +bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_get_counter("rpc_kv","alter_obj_store_info_get_counter"); +bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_put_counter("rpc_kv","alter_obj_store_info_put_counter"); +bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_del_counter("rpc_kv","alter_obj_store_info_del_counter"); // update_ak_sk -bvar::Adder g_bvar_rpc_kv_update_ak_sk_read_counter("rpc_kv","update_ak_sk_read_counter"); -bvar::Adder g_bvar_rpc_kv_update_ak_sk_write_counter("rpc_kv","update_ak_sk_write_counter"); +bvar::Adder g_bvar_rpc_kv_update_ak_sk_get_counter("rpc_kv","update_ak_sk_get_counter"); +bvar::Adder g_bvar_rpc_kv_update_ak_sk_put_counter("rpc_kv","update_ak_sk_put_counter"); +bvar::Adder g_bvar_rpc_kv_update_ak_sk_del_counter("rpc_kv","update_ak_sk_del_counter"); // create_instance -bvar::Adder g_bvar_rpc_kv_create_instance_read_counter("rpc_kv","create_instance_read_counter"); -bvar::Adder g_bvar_rpc_kv_create_instance_write_counter("rpc_kv","create_instance_write_counter"); +bvar::Adder g_bvar_rpc_kv_create_instance_get_counter("rpc_kv","create_instance_get_counter"); +bvar::Adder g_bvar_rpc_kv_create_instance_put_counter("rpc_kv","create_instance_put_counter"); +bvar::Adder g_bvar_rpc_kv_create_instance_del_counter("rpc_kv","create_instance_del_counter"); // get_instance -bvar::Adder g_bvar_rpc_kv_get_instance_read_counter("rpc_kv","get_instance_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_instance_write_counter("rpc_kv","get_instance_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_instance_get_counter("rpc_kv","get_instance_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_instance_put_counter("rpc_kv","get_instance_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_instance_del_counter("rpc_kv","get_instance_del_counter"); // alter_cluster -bvar::Adder g_bvar_rpc_kv_alter_cluster_read_counter("rpc_kv","alter_cluster_read_counter"); -bvar::Adder g_bvar_rpc_kv_alter_cluster_write_counter("rpc_kv","alter_cluster_write_counter"); +bvar::Adder g_bvar_rpc_kv_alter_cluster_get_counter("rpc_kv","alter_cluster_get_counter"); +bvar::Adder g_bvar_rpc_kv_alter_cluster_put_counter("rpc_kv","alter_cluster_put_counter"); +bvar::Adder g_bvar_rpc_kv_alter_cluster_del_counter("rpc_kv","alter_cluster_del_counter"); // get_cluster -bvar::Adder g_bvar_rpc_kv_get_cluster_read_counter("rpc_kv","get_cluster_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_cluster_write_counter("rpc_kv","get_cluster_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_cluster_get_counter("rpc_kv","get_cluster_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_cluster_put_counter("rpc_kv","get_cluster_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_cluster_del_counter("rpc_kv","get_cluster_del_counter"); // create_stage -bvar::Adder g_bvar_rpc_kv_create_stage_read_counter("rpc_kv","create_stage_read_counter"); -bvar::Adder g_bvar_rpc_kv_create_stage_write_counter("rpc_kv","create_stage_write_counter"); +bvar::Adder g_bvar_rpc_kv_create_stage_get_counter("rpc_kv","create_stage_get_counter"); +bvar::Adder g_bvar_rpc_kv_create_stage_put_counter("rpc_kv","create_stage_put_counter"); +bvar::Adder g_bvar_rpc_kv_create_stage_del_counter("rpc_kv","create_stage_del_counter"); // get_stage -bvar::Adder g_bvar_rpc_kv_get_stage_read_counter("rpc_kv","get_stage_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_stage_write_counter("rpc_kv","get_stage_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_stage_get_counter("rpc_kv","get_stage_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_stage_put_counter("rpc_kv","get_stage_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_stage_del_counter("rpc_kv","get_stage_del_counter"); // get_iam -bvar::Adder g_bvar_rpc_kv_get_iam_read_counter("rpc_kv","get_iam_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_iam_write_counter("rpc_kv","get_iam_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_iam_get_counter("rpc_kv","get_iam_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_iam_put_counter("rpc_kv","get_iam_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_iam_del_counter("rpc_kv","get_iam_del_counter"); // alter_iam -bvar::Adder g_bvar_rpc_kv_alter_iam_read_counter("rpc_kv","alter_iam_read_counter"); -bvar::Adder g_bvar_rpc_kv_alter_iam_write_counter("rpc_kv","alter_iam_write_counter"); +bvar::Adder g_bvar_rpc_kv_alter_iam_get_counter("rpc_kv","alter_iam_get_counter"); +bvar::Adder g_bvar_rpc_kv_alter_iam_put_counter("rpc_kv","alter_iam_put_counter"); +bvar::Adder g_bvar_rpc_kv_alter_iam_del_counter("rpc_kv","alter_iam_del_counter"); // alter_ram_user -bvar::Adder g_bvar_rpc_kv_alter_ram_user_read_counter("rpc_kv","alter_ram_user_read_counter"); -bvar::Adder g_bvar_rpc_kv_alter_ram_user_write_counter("rpc_kv","alter_ram_user_write_counter"); +bvar::Adder g_bvar_rpc_kv_alter_ram_user_get_counter("rpc_kv","alter_ram_user_get_counter"); +bvar::Adder g_bvar_rpc_kv_alter_ram_user_put_counter("rpc_kv","alter_ram_user_put_counter"); +bvar::Adder g_bvar_rpc_kv_alter_ram_user_del_counter("rpc_kv","alter_ram_user_del_counter"); // begin_copy -bvar::Adder g_bvar_rpc_kv_begin_copy_read_counter("rpc_kv","begin_copy_read_counter"); -bvar::Adder g_bvar_rpc_kv_begin_copy_write_counter("rpc_kv","begin_copy_write_counter"); +bvar::Adder g_bvar_rpc_kv_begin_copy_get_counter("rpc_kv","begin_copy_get_counter"); +bvar::Adder g_bvar_rpc_kv_begin_copy_put_counter("rpc_kv","begin_copy_put_counter"); +bvar::Adder g_bvar_rpc_kv_begin_copy_del_counter("rpc_kv","begin_copy_del_counter"); // finish_copy -bvar::Adder g_bvar_rpc_kv_finish_copy_read_counter("rpc_kv","finish_copy_read_counter"); -bvar::Adder g_bvar_rpc_kv_finish_copy_write_counter("rpc_kv","finish_copy_write_counter"); +bvar::Adder g_bvar_rpc_kv_finish_copy_get_counter("rpc_kv","finish_copy_get_counter"); +bvar::Adder g_bvar_rpc_kv_finish_copy_put_counter("rpc_kv","finish_copy_put_counter"); +bvar::Adder g_bvar_rpc_kv_finish_copy_del_counter("rpc_kv","finish_copy_del_counter"); // get_copy_job -bvar::Adder g_bvar_rpc_kv_get_copy_job_read_counter("rpc_kv","get_copy_job_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_copy_job_write_counter("rpc_kv","get_copy_job_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_copy_job_get_counter("rpc_kv","get_copy_job_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_copy_job_put_counter("rpc_kv","get_copy_job_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_copy_job_del_counter("rpc_kv","get_copy_job_del_counter"); // get_copy_files -bvar::Adder g_bvar_rpc_kv_get_copy_files_read_counter("rpc_kv","get_copy_files_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_copy_files_write_counter("rpc_kv","get_copy_files_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_copy_files_get_counter("rpc_kv","get_copy_files_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_copy_files_put_counter("rpc_kv","get_copy_files_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_copy_files_del_counter("rpc_kv","get_copy_files_del_counter"); // filter_copy_files -bvar::Adder g_bvar_rpc_kv_filter_copy_files_read_counter("rpc_kv","filter_copy_files_read_counter"); -bvar::Adder g_bvar_rpc_kv_filter_copy_files_write_counter("rpc_kv","filter_copy_files_write_counter"); +bvar::Adder g_bvar_rpc_kv_filter_copy_files_get_counter("rpc_kv","filter_copy_files_get_counter"); +bvar::Adder g_bvar_rpc_kv_filter_copy_files_put_counter("rpc_kv","filter_copy_files_put_counter"); +bvar::Adder g_bvar_rpc_kv_filter_copy_files_del_counter("rpc_kv","filter_copy_files_del_counter"); // get_cluster_status -bvar::Adder g_bvar_rpc_kv_get_cluster_status_read_counter("rpc_kv","get_cluster_status_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_cluster_status_write_counter("rpc_kv","get_cluster_status_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_cluster_status_get_counter("rpc_kv","get_cluster_status_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_cluster_status_put_counter("rpc_kv","get_cluster_status_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_cluster_status_del_counter("rpc_kv","get_cluster_status_del_counter"); // begin_txn -bvar::Adder g_bvar_rpc_kv_begin_txn_read_counter("rpc_kv","begin_txn_read_counter"); -bvar::Adder g_bvar_rpc_kv_begin_txn_write_counter("rpc_kv","begin_txn_write_counter"); +bvar::Adder g_bvar_rpc_kv_begin_txn_get_counter("rpc_kv","begin_txn_get_counter"); +bvar::Adder g_bvar_rpc_kv_begin_txn_put_counter("rpc_kv","begin_txn_put_counter"); +bvar::Adder g_bvar_rpc_kv_begin_txn_del_counter("rpc_kv","begin_txn_del_counter"); // precommit_txn -bvar::Adder g_bvar_rpc_kv_precommit_txn_read_counter("rpc_kv","precommit_txn_read_counter"); -bvar::Adder g_bvar_rpc_kv_precommit_txn_write_counter("rpc_kv","precommit_txn_write_counter"); +bvar::Adder g_bvar_rpc_kv_precommit_txn_get_counter("rpc_kv","precommit_txn_get_counter"); +bvar::Adder g_bvar_rpc_kv_precommit_txn_put_counter("rpc_kv","precommit_txn_put_counter"); +bvar::Adder g_bvar_rpc_kv_precommit_txn_del_counter("rpc_kv","precommit_txn_del_counter"); // get_rl_task_commit_attach -bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_read_counter("rpc_kv","get_rl_task_commit_attach_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_write_counter("rpc_kv","get_rl_task_commit_attach_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter("rpc_kv","get_rl_task_commit_attach_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_put_counter("rpc_kv","get_rl_task_commit_attach_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_del_counter("rpc_kv","get_rl_task_commit_attach_del_counter"); // reset_rl_progress -bvar::Adder g_bvar_rpc_kv_reset_rl_progress_read_counter("rpc_kv","reset_rl_progress_read_counter"); -bvar::Adder g_bvar_rpc_kv_reset_rl_progress_write_counter("rpc_kv","reset_rl_progress_write_counter"); +bvar::Adder g_bvar_rpc_kv_reset_rl_progress_get_counter("rpc_kv","reset_rl_progress_get_counter"); +bvar::Adder g_bvar_rpc_kv_reset_rl_progress_put_counter("rpc_kv","reset_rl_progress_put_counter"); +bvar::Adder g_bvar_rpc_kv_reset_rl_progress_del_counter("rpc_kv","reset_rl_progress_del_counter"); // commit_txn -bvar::Adder g_bvar_rpc_kv_commit_txn_read_counter("rpc_kv","commit_txn_read_counter"); -bvar::Adder g_bvar_rpc_kv_commit_txn_write_counter("rpc_kv","commit_txn_write_counter"); +bvar::Adder g_bvar_rpc_kv_commit_txn_get_counter("rpc_kv","commit_txn_get_counter"); +bvar::Adder g_bvar_rpc_kv_commit_txn_put_counter("rpc_kv","commit_txn_put_counter"); +bvar::Adder g_bvar_rpc_kv_commit_txn_del_counter("rpc_kv","commit_txn_del_counter"); // abort_txn -bvar::Adder g_bvar_rpc_kv_abort_txn_read_counter("rpc_kv","abort_txn_read_counter"); -bvar::Adder g_bvar_rpc_kv_abort_txn_write_counter("rpc_kv","abort_txn_write_counter"); +bvar::Adder g_bvar_rpc_kv_abort_txn_get_counter("rpc_kv","abort_txn_get_counter"); +bvar::Adder g_bvar_rpc_kv_abort_txn_put_counter("rpc_kv","abort_txn_put_counter"); +bvar::Adder g_bvar_rpc_kv_abort_txn_del_counter("rpc_kv","abort_txn_del_counter"); // get_txn -bvar::Adder g_bvar_rpc_kv_get_txn_read_counter("rpc_kv","get_txn_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_txn_write_counter("rpc_kv","get_txn_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_txn_get_counter("rpc_kv","get_txn_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_txn_put_counter("rpc_kv","get_txn_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_txn_del_counter("rpc_kv","get_txn_del_counter"); // get_current_max_txn_id -bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_read_counter("rpc_kv","get_current_max_txn_id_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_write_counter("rpc_kv","get_current_max_txn_id_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_get_counter("rpc_kv","get_current_max_txn_id_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_put_counter("rpc_kv","get_current_max_txn_id_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_del_counter("rpc_kv","get_current_max_txn_id_del_counter"); // begin_sub_txn -bvar::Adder g_bvar_rpc_kv_begin_sub_txn_read_counter("rpc_kv","begin_sub_txn_read_counter"); -bvar::Adder g_bvar_rpc_kv_begin_sub_txn_write_counter("rpc_kv","begin_sub_txn_write_counter"); +bvar::Adder g_bvar_rpc_kv_begin_sub_txn_get_counter("rpc_kv","begin_sub_txn_get_counter"); +bvar::Adder g_bvar_rpc_kv_begin_sub_txn_put_counter("rpc_kv","begin_sub_txn_put_counter"); +bvar::Adder g_bvar_rpc_kv_begin_sub_txn_del_counter("rpc_kv","begin_sub_txn_del_counter"); // abort_sub_txn -bvar::Adder g_bvar_rpc_kv_abort_sub_txn_read_counter("rpc_kv","abort_sub_txn_read_counter"); -bvar::Adder g_bvar_rpc_kv_abort_sub_txn_write_counter("rpc_kv","abort_sub_txn_write_counter"); +bvar::Adder g_bvar_rpc_kv_abort_sub_txn_get_counter("rpc_kv","abort_sub_txn_get_counter"); +bvar::Adder g_bvar_rpc_kv_abort_sub_txn_put_counter("rpc_kv","abort_sub_txn_put_counter"); +bvar::Adder g_bvar_rpc_kv_abort_sub_txn_del_counter("rpc_kv","abort_sub_txn_del_counter"); // abort_txn_with_coordinator -bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_read_counter("rpc_kv","abort_txn_with_coordinator_read_counter"); -bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_write_counter("rpc_kv","abort_txn_with_coordinator_write_counter"); +bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter("rpc_kv","abort_txn_with_coordinator_get_counter"); +bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_put_counter("rpc_kv","abort_txn_with_coordinator_put_counter"); +bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_del_counter("rpc_kv","abort_txn_with_coordinator_del_counter"); // check_txn_conflict -bvar::Adder g_bvar_rpc_kv_check_txn_conflict_read_counter("rpc_kv","check_txn_conflict_read_counter"); -bvar::Adder g_bvar_rpc_kv_check_txn_conflict_write_counter("rpc_kv","check_txn_conflict_write_counter"); +bvar::Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv","check_txn_conflict_get_counter"); +bvar::Adder g_bvar_rpc_kv_check_txn_conflict_put_counter("rpc_kv","check_txn_conflict_put_counter"); +bvar::Adder g_bvar_rpc_kv_check_txn_conflict_del_counter("rpc_kv","check_txn_conflict_del_counter"); // clean_txn_label -bvar::Adder g_bvar_rpc_kv_clean_txn_label_read_counter("rpc_kv","clean_txn_label_read_counter"); -bvar::Adder g_bvar_rpc_kv_clean_txn_label_write_counter("rpc_kv","clean_txn_label_write_counter"); +bvar::Adder g_bvar_rpc_kv_clean_txn_label_get_counter("rpc_kv","clean_txn_label_get_counter"); +bvar::Adder g_bvar_rpc_kv_clean_txn_label_put_counter("rpc_kv","clean_txn_label_put_counter"); +bvar::Adder g_bvar_rpc_kv_clean_txn_label_del_counter("rpc_kv","clean_txn_label_del_counter"); // get_txn_id -bvar::Adder g_bvar_rpc_kv_get_txn_id_read_counter("rpc_kv","get_txn_id_read_counter"); -bvar::Adder g_bvar_rpc_kv_get_txn_id_write_counter("rpc_kv","get_txn_id_write_counter"); +bvar::Adder g_bvar_rpc_kv_get_txn_id_get_counter("rpc_kv","get_txn_id_get_counter"); +bvar::Adder g_bvar_rpc_kv_get_txn_id_put_counter("rpc_kv","get_txn_id_put_counter"); +bvar::Adder g_bvar_rpc_kv_get_txn_id_del_counter("rpc_kv","get_txn_id_del_counter"); // clang-format on diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 9b0dda69329bec..af2e8455770993 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -256,119 +256,177 @@ extern BvarStatusWithTag g_bvar_inverted_checker_delete_bitmaps_scanned extern BvarStatusWithTag g_bvar_max_rowsets_with_useless_delete_bitmap_version; // rpc kv -extern bvar::Adder g_bvar_rpc_kv_get_rowset_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_rowset_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_version_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_version_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_write_counter; -extern bvar::Adder g_bvar_rpc_kv_create_tablets_read_counter; -extern bvar::Adder g_bvar_rpc_kv_create_tablets_write_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tablet_read_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tablet_write_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_read_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_tablet_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_tablet_write_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_read_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_write_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_rowset_read_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_rowset_write_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_read_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_write_counter; -extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_read_counter; -extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_write_counter; -extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_read_counter; -extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_write_counter; -extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_read_counter; -extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_write_counter; -extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_read_counter; -extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_write_counter; -extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_read_counter; -extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_write_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_index_read_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_index_write_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_index_read_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_index_write_counter; -extern bvar::Adder g_bvar_rpc_kv_drop_index_read_counter; -extern bvar::Adder g_bvar_rpc_kv_drop_index_write_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_partition_read_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_partition_write_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_partition_read_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_partition_write_counter; -extern bvar::Adder g_bvar_rpc_kv_drop_partition_read_counter; -extern bvar::Adder g_bvar_rpc_kv_drop_partition_write_counter; -extern bvar::Adder g_bvar_rpc_kv_check_kv_read_counter; -extern bvar::Adder g_bvar_rpc_kv_check_kv_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_write_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_read_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_write_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_read_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_write_counter; -extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_read_counter; -extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_write_counter; -extern bvar::Adder g_bvar_rpc_kv_create_instance_read_counter; -extern bvar::Adder g_bvar_rpc_kv_create_instance_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_instance_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_instance_write_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_cluster_read_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_cluster_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_cluster_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_cluster_write_counter; -extern bvar::Adder g_bvar_rpc_kv_create_stage_read_counter; -extern bvar::Adder g_bvar_rpc_kv_create_stage_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_stage_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_stage_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_iam_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_iam_write_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_iam_read_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_iam_write_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_read_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_write_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_copy_read_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_copy_write_counter; -extern bvar::Adder g_bvar_rpc_kv_finish_copy_read_counter; -extern bvar::Adder g_bvar_rpc_kv_finish_copy_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_copy_job_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_copy_job_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_copy_files_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_copy_files_write_counter; -extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_read_counter; -extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_write_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_txn_read_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_txn_write_counter; -extern bvar::Adder g_bvar_rpc_kv_precommit_txn_read_counter; -extern bvar::Adder g_bvar_rpc_kv_precommit_txn_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_write_counter; -extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_read_counter; -extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_write_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_txn_read_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_txn_write_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_txn_read_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_txn_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_txn_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_txn_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_write_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_read_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_write_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_read_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_write_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_read_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_write_counter; -extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_read_counter; -extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_write_counter; -extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_read_counter; -extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_write_counter; -extern bvar::Adder g_bvar_rpc_kv_get_txn_id_read_counter; -extern bvar::Adder g_bvar_rpc_kv_get_txn_id_write_counter; +extern bvar::Adder g_bvar_rpc_kv_get_rowset_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_rowset_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_rowset_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_version_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_version_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_version_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_del_counter; +extern bvar::Adder g_bvar_rpc_kv_create_tablets_get_counter; +extern bvar::Adder g_bvar_rpc_kv_create_tablets_put_counter; +extern bvar::Adder g_bvar_rpc_kv_create_tablets_del_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tablet_get_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tablet_put_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tablet_del_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_get_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_put_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_tablet_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_tablet_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_tablet_del_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_get_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_put_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_del_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_rowset_get_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_rowset_put_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_rowset_del_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_get_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_put_counter; +extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_del_counter; +extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_get_counter; +extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_put_counter; +extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter; +extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter; +extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter; +extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter; +extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_get_counter; +extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_put_counter; +extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_del_counter; +extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_get_counter; +extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_put_counter; +extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_del_counter; +extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_get_counter; +extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_put_counter; +extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_del_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_index_get_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_index_put_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_index_del_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_index_get_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_index_put_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_index_del_counter; +extern bvar::Adder g_bvar_rpc_kv_drop_index_get_counter; +extern bvar::Adder g_bvar_rpc_kv_drop_index_put_counter; +extern bvar::Adder g_bvar_rpc_kv_drop_index_del_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_partition_get_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_partition_put_counter; +extern bvar::Adder g_bvar_rpc_kv_prepare_partition_del_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_partition_get_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_partition_put_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_partition_del_counter; +extern bvar::Adder g_bvar_rpc_kv_drop_partition_get_counter; +extern bvar::Adder g_bvar_rpc_kv_drop_partition_put_counter; +extern bvar::Adder g_bvar_rpc_kv_drop_partition_del_counter; +extern bvar::Adder g_bvar_rpc_kv_check_kv_get_counter; +extern bvar::Adder g_bvar_rpc_kv_check_kv_put_counter; +extern bvar::Adder g_bvar_rpc_kv_check_kv_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_del_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_get_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_put_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_del_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_get_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_put_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_del_counter; +extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_get_counter; +extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_put_counter; +extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_del_counter; +extern bvar::Adder g_bvar_rpc_kv_create_instance_get_counter; +extern bvar::Adder g_bvar_rpc_kv_create_instance_put_counter; +extern bvar::Adder g_bvar_rpc_kv_create_instance_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_instance_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_instance_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_instance_del_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_cluster_get_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_cluster_put_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_cluster_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_cluster_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_cluster_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_cluster_del_counter; +extern bvar::Adder g_bvar_rpc_kv_create_stage_get_counter; +extern bvar::Adder g_bvar_rpc_kv_create_stage_put_counter; +extern bvar::Adder g_bvar_rpc_kv_create_stage_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_stage_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_stage_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_stage_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_iam_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_iam_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_iam_del_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_iam_get_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_iam_put_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_iam_del_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_get_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_put_counter; +extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_del_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_copy_get_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_copy_put_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_copy_del_counter; +extern bvar::Adder g_bvar_rpc_kv_finish_copy_get_counter; +extern bvar::Adder g_bvar_rpc_kv_finish_copy_put_counter; +extern bvar::Adder g_bvar_rpc_kv_finish_copy_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_copy_job_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_copy_job_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_copy_job_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_copy_files_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_copy_files_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_copy_files_del_counter; +extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_get_counter; +extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_put_counter; +extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_del_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_txn_get_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_txn_put_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_txn_del_counter; +extern bvar::Adder g_bvar_rpc_kv_precommit_txn_get_counter; +extern bvar::Adder g_bvar_rpc_kv_precommit_txn_put_counter; +extern bvar::Adder g_bvar_rpc_kv_precommit_txn_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_del_counter; +extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_get_counter; +extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_put_counter; +extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_del_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_txn_get_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_txn_put_counter; +extern bvar::Adder g_bvar_rpc_kv_commit_txn_del_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_txn_get_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_txn_put_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_txn_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_txn_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_txn_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_txn_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_del_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_get_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_put_counter; +extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_del_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_get_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_put_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_del_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_put_counter; +extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_del_counter; +extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_get_counter; +extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_put_counter; +extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_del_counter; +extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_get_counter; +extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_put_counter; +extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_del_counter; +extern bvar::Adder g_bvar_rpc_kv_get_txn_id_get_counter; +extern bvar::Adder g_bvar_rpc_kv_get_txn_id_put_counter; +extern bvar::Adder g_bvar_rpc_kv_get_txn_id_del_counter; diff --git a/cloud/src/common/stats.h b/cloud/src/common/stats.h index b9ff4e3a325cac..feda1552969d7b 100644 --- a/cloud/src/common/stats.h +++ b/cloud/src/common/stats.h @@ -23,11 +23,13 @@ namespace doris::cloud { struct KVStats { - KVStats(bvar::Adder& read_counter, bvar::Adder& write_counter) - : read_counter(read_counter), write_counter(write_counter) {} + KVStats(bvar::Adder& get_counter, bvar::Adder& put_counter, + bvar::Adder& del_counter) + : get_counter(get_counter), put_counter(put_counter), del_counter(del_counter) {} - bvar::Adder& read_counter; - bvar::Adder& write_counter; + bvar::Adder& get_counter; + bvar::Adder& put_counter; + bvar::Adder& del_counter; }; } // namespace doris::cloud \ No newline at end of file diff --git a/cloud/src/meta-service/mem_txn_kv.cpp b/cloud/src/meta-service/mem_txn_kv.cpp index 7eef0a5dd50a90..bc7b92efdb2853 100644 --- a/cloud/src/meta-service/mem_txn_kv.cpp +++ b/cloud/src/meta-service/mem_txn_kv.cpp @@ -248,7 +248,7 @@ void Transaction::put(std::string_view key, std::string_view val) { writes_.insert_or_assign(k, v); op_list_.emplace_back(ModifyOpType::PUT, k, v); ++num_put_keys_; - kv_->write_count_++; + kv_->put_count_++; put_bytes_ += key.size() + val.size(); approximate_bytes_ += key.size() + val.size(); } @@ -302,7 +302,7 @@ TxnErrorCode Transaction::inner_get(const std::string& key, std::string* val, bo } } num_get_keys_++; - kv_->read_count_++; + kv_->get_count_++; return TxnErrorCode::TXN_OK; } @@ -348,14 +348,14 @@ TxnErrorCode Transaction::inner_get(const std::string& begin, const std::string& std::vector> kv_list(kv_map.begin(), kv_map.end()); num_get_keys_ += kv_list.size(); - kv_->read_count_ += kv_list.size(); + kv_->get_count_ += kv_list.size(); *iter = std::make_unique(std::move(kv_list), more); return TxnErrorCode::TXN_OK; } void Transaction::atomic_set_ver_key(std::string_view key_prefix, std::string_view val) { std::lock_guard l(lock_); - kv_->write_count_++; + kv_->put_count_++; std::string k(key_prefix.data(), key_prefix.size()); std::string v(val.data(), val.size()); unreadable_keys_.insert(k); @@ -368,7 +368,7 @@ void Transaction::atomic_set_ver_key(std::string_view key_prefix, std::string_vi void Transaction::atomic_set_ver_value(std::string_view key, std::string_view value) { std::lock_guard l(lock_); - kv_->write_count_++; + kv_->put_count_++; std::string k(key.data(), key.size()); std::string v(value.data(), value.size()); unreadable_keys_.insert(k); @@ -384,7 +384,7 @@ void Transaction::atomic_add(std::string_view key, int64_t to_add) { std::string v(sizeof(to_add), '\0'); memcpy(v.data(), &to_add, sizeof(to_add)); std::lock_guard l(lock_); - kv_->write_count_++; + kv_->put_count_++; op_list_.emplace_back(ModifyOpType::ATOMIC_ADD, std::move(k), std::move(v)); ++num_put_keys_; @@ -403,7 +403,7 @@ bool Transaction::decode_atomic_int(std::string_view data, int64_t* val) { void Transaction::remove(std::string_view key) { std::lock_guard l(lock_); - kv_->write_count_++; + kv_->del_count_++; std::string k(key.data(), key.size()); writes_.erase(k); std::string end_key = k; @@ -430,7 +430,7 @@ void Transaction::remove(std::string_view begin, std::string_view end) { remove_ranges_.emplace_back(begin_k, end_k); op_list_.emplace_back(ModifyOpType::REMOVE_RANGE, begin_k, end_k); } - kv_->write_count_ += 2; + kv_->del_count_ += 2; // same as normal txn num_del_keys_ += 2; delete_bytes_ += begin.size() + end.size(); @@ -491,6 +491,7 @@ TxnErrorCode Transaction::batch_get(std::vector>* res auto ret = inner_get(k, &val, opts.snapshot); ret == TxnErrorCode::TXN_OK ? res->push_back(val) : res->push_back(std::nullopt); } + kv_->get_count_ += keys.size(); num_get_keys_ += keys.size(); return TxnErrorCode::TXN_OK; } diff --git a/cloud/src/meta-service/mem_txn_kv.h b/cloud/src/meta-service/mem_txn_kv.h index 9f53fed2c792a9..1046adcee1ce33 100644 --- a/cloud/src/meta-service/mem_txn_kv.h +++ b/cloud/src/meta-service/mem_txn_kv.h @@ -30,7 +30,6 @@ #include #include -#include "common/stats.h" #include "meta-service/txn_kv_error.h" #include "txn_kv.h" @@ -59,8 +58,9 @@ class MemTxnKv : public TxnKv, public std::enable_shared_from_this { TxnErrorCode get_kv(const std::string& begin, const std::string& end, int64_t version, int limit, bool* more, std::map* kv_list); - int64_t read_count_ {}; - int64_t write_count_ {}; + int64_t get_count_ {}; + int64_t put_count_ {}; + int64_t del_count_ {}; private: using OpTuple = std::tuple; diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index b53bdf172b85a2..cf5503fa1262dc 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -366,8 +366,9 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr break; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.read_counter << txn->num_get_keys(); - stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); for (size_t i = response->versions_size(); i < num_acquired; i += BATCH_SIZE) { size_t limit = (i + BATCH_SIZE < num_acquired) ? i + BATCH_SIZE : num_acquired; @@ -467,8 +468,9 @@ void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.read_counter << txn->num_get_keys(); - stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); std::string rs_key, rs_val; @@ -608,8 +610,9 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.read_counter << txn0->num_get_keys(); - stats.write_counter << txn0->num_del_keys() + txn0->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); InstanceKeyInfo key_info {instance_id}; @@ -1738,8 +1741,9 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.read_counter << txn->num_get_keys(); - stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); TabletIndexPB idx; // Get tablet id index from kv @@ -1912,8 +1916,9 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.read_counter << txn->num_get_keys(); - stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); txn.reset(nullptr); }); if (!(/* idx.has_db_id() && */ idx.has_table_id() && idx.has_index_id() && @@ -2361,8 +2366,9 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont err = txn->commit(); TEST_SYNC_POINT_CALLBACK("update_delete_bitmap:commit:err", request->initiator(), i, &err); - stats.read_counter << txn->num_get_keys(); - stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); total_txn_put_keys += txn->num_put_keys(); total_txn_put_bytes += txn->put_bytes(); total_txn_size += txn->approximate_bytes(); @@ -2587,8 +2593,9 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.read_counter << txn->num_get_keys(); - stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); MetaDeleteBitmapInfo start_key_info {instance_id, tablet_id, rowset_ids[i], begin_versions[i], 0}; @@ -2708,8 +2715,9 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.read_counter << txn->num_get_keys(); - stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); TabletIndexPB idx(request->idx()); TabletStatsPB tablet_stat; @@ -2777,7 +2785,7 @@ bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std:: const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, std::string& instance_id, std::string& lock_key, - std::string lock_use_version, KVStats* stats) { + std::string lock_use_version, KVStats& stats) { bool require_tablet_stats = request->has_require_compaction_stats() ? request->require_compaction_stats() : false; if (!require_tablet_stats) return true; @@ -2798,8 +2806,9 @@ bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std:: return false; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats->read_counter << txn->num_get_keys(); - stats->write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); auto table_id = request->table_id(); std::stringstream ss; @@ -2976,7 +2985,7 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2( const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss, - KVStats* stats) { + KVStats& stats) { VLOG_DEBUG << "get delete bitmap update lock in v2 for table=" << request->table_id() << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator(); auto table_id = request->table_id(); @@ -2994,8 +3003,9 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats->read_counter << txn->num_get_keys(); - stats->write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); std::string lock_val; DeleteBitmapUpdateLockPB lock_info; @@ -3220,7 +3230,7 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v1( const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss, - KVStats* stats) { + KVStats& stats) { VLOG_DEBUG << "get delete bitmap update lock in v1 for table=" << request->table_id() << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator(); std::unique_ptr txn; @@ -3231,8 +3241,9 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v1( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats->read_counter << txn->num_get_keys(); - stats->write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); auto table_id = request->table_id(); std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}); @@ -3310,7 +3321,7 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v2( const RemoveDeleteBitmapUpdateLockRequest* request, RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss, - KVStats* stats) { + KVStats& stats) { VLOG_DEBUG << "remove delete bitmap update lock in v2 for table=" << request->table_id() << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator(); std::unique_ptr txn; @@ -3321,8 +3332,9 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v2( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats->read_counter << txn->num_get_keys(); - stats->write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); if (request->lock_id() == COMPACTION_DELETE_BITMAP_LOCK_ID) { std::string tablet_compaction_key = @@ -3394,7 +3406,7 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v1( const RemoveDeleteBitmapUpdateLockRequest* request, RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss, - KVStats* stats) { + KVStats& stats) { VLOG_DEBUG << "remove delete bitmap update lock in v1 for table=" << request->table_id() << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator(); std::unique_ptr txn; @@ -3405,8 +3417,9 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v1( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats->read_counter << txn->num_get_keys(); - stats->write_counter << txn->num_del_keys() + txn->num_put_keys(); + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); }); std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, request->table_id(), -1}); @@ -3484,10 +3497,10 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl << " use_version=" << use_version; if (use_version == "v2") { get_delete_bitmap_update_lock_v2(controller, request, response, done, instance_id, code, - msg, ss, &stats); + msg, ss, stats); } else { get_delete_bitmap_update_lock_v1(controller, request, response, done, instance_id, code, - msg, ss, &stats); + msg, ss, stats); } } @@ -3515,10 +3528,10 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock( << " use_version=" << use_version; if (use_version == "v2") { remove_delete_bitmap_update_lock_v2(controller, request, response, done, instance_id, code, - msg, ss, &stats); + msg, ss, stats); } else { remove_delete_bitmap_update_lock_v1(controller, request, response, done, instance_id, code, - msg, ss, &stats); + msg, ss, stats); } } diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index 5bff9710be5794..df681bd05a8d63 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -330,21 +330,21 @@ class MetaServiceImpl : public cloud::MetaService { const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, std::string& instance_id, std::string& lock_key, - std::string lock_use_version, KVStats* stats); + std::string lock_use_version, KVStats& stats); void get_delete_bitmap_update_lock_v2(google::protobuf::RpcController* controller, const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, - std::string& msg, std::stringstream& ss, KVStats* stats); + std::string& msg, std::stringstream& ss, KVStats& stats); void get_delete_bitmap_update_lock_v1(google::protobuf::RpcController* controller, const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, - std::string& msg, std::stringstream& ss, KVStats* stats); + std::string& msg, std::stringstream& ss, KVStats& stats); void remove_delete_bitmap_update_lock_v2(google::protobuf::RpcController* controller, const RemoveDeleteBitmapUpdateLockRequest* request, @@ -352,7 +352,7 @@ class MetaServiceImpl : public cloud::MetaService { ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss, - KVStats* stats); + KVStats& stats); void remove_delete_bitmap_update_lock_v1(google::protobuf::RpcController* controller, const RemoveDeleteBitmapUpdateLockRequest* request, @@ -360,7 +360,7 @@ class MetaServiceImpl : public cloud::MetaService { ::google::protobuf::Closure* done, std::string& instance_id, MetaServiceCode& code, std::string& msg, std::stringstream& ss, - KVStats* stats); + KVStats& stats); std::shared_ptr txn_kv_; std::shared_ptr resource_mgr_; diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index cb50aa5cf67086..cce7a46add84b6 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -237,8 +237,9 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { [[maybe_unused]] std::string msg; \ [[maybe_unused]] std::string instance_id; \ [[maybe_unused]] bool drop_request = false; \ - [[maybe_unused]] KVStats stats(g_bvar_rpc_kv_##func_name##_read_counter, \ - g_bvar_rpc_kv_##func_name##_write_counter); \ + [[maybe_unused]] KVStats stats(g_bvar_rpc_kv_##func_name##_get_counter, \ + g_bvar_rpc_kv_##func_name##_put_counter, \ + g_bvar_rpc_kv_##func_name##_del_counter); \ std::unique_ptr> defer_status((int*)0x01, [&](int*) { \ response->mutable_status()->set_code(code); \ response->mutable_status()->set_msg(msg); \ @@ -246,10 +247,11 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { closure_guard.reset(nullptr); \ if (config::use_detailed_metrics && !instance_id.empty() && !drop_request) { \ g_bvar_ms_##func_name.put(instance_id, sw.elapsed_us()); \ - if (txn != nullptr) { \ - stats.read_counter << txn->num_get_keys(); \ - stats.write_counter << txn->num_del_keys() + txn->num_put_keys(); \ - } \ + } \ + if (txn != nullptr) { \ + stats.get_counter << txn->num_get_keys(); \ + stats.put_counter << txn->num_put_keys(); \ + stats.del_counter << txn->num_del_keys(); \ } \ }); @@ -279,7 +281,8 @@ int decrypt_instance_info(InstanceInfoPB& instance, const std::string& instance_ /** * Notifies other metaservice to refresh instance */ -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id); +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, + KVStats* stats); void get_tablet_idx(MetaServiceCode& code, std::string& msg, Transaction* txn, const std::string& instance_id, int64_t tablet_id, TabletIndexPB& tablet_idx); diff --git a/cloud/src/meta-service/meta_service_partition.cpp b/cloud/src/meta-service/meta_service_partition.cpp index ea629fd0f4869f..99d544cd931628 100644 --- a/cloud/src/meta-service/meta_service_partition.cpp +++ b/cloud/src/meta-service/meta_service_partition.cpp @@ -18,7 +18,10 @@ #include #include +#include + #include "common/logging.h" +#include "common/stats.h" #include "meta-service/keys.h" #include "meta-service/meta_service_helper.h" #include "meta-service/txn_kv_error.h" @@ -617,7 +620,7 @@ void MetaServiceImpl::drop_partition(::google::protobuf::RpcController* controll void check_create_table(std::string instance_id, std::shared_ptr txn_kv, const CheckKVRequest* request, CheckKVResponse* response, - MetaServiceCode* code, std::string* msg, + MetaServiceCode* code, std::string* msg, KVStats& stats, check_create_table_type get_check_info) { StopWatch watch; std::unique_ptr txn; @@ -627,6 +630,8 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, *msg = "failed to create txn"; return; } + std::unique_ptr> defer_stats( + (int*)0x01, [&](int*) { stats.get_counter << txn->num_get_keys(); }); auto& [keys, hint, key_func] = get_check_info(request); if (keys.empty()) { *code = MetaServiceCode::INVALID_ARGUMENT; @@ -646,6 +651,7 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, *msg = "prepare and commit rpc not match, recycle key remained"; return; } else if (err == TxnErrorCode::TXN_TOO_OLD) { + stats.get_counter << txn->num_get_keys(); // separate it to several txn for rubustness txn.reset(); TxnErrorCode err = txn_kv->create_txn(&txn); @@ -691,7 +697,7 @@ void MetaServiceImpl::check_kv(::google::protobuf::RpcController* controller, RPC_RATE_LIMIT(check_kv); switch (request->op()) { case CheckKVRequest::CREATE_INDEX_AFTER_FE_COMMIT: { - check_create_table(instance_id, txn_kv_, request, response, &code, &msg, + check_create_table(instance_id, txn_kv_, request, response, &code, &msg, stats, [](const CheckKVRequest* request) { return std::make_tuple( request->check_keys().index_ids(), "index", @@ -703,7 +709,7 @@ void MetaServiceImpl::check_kv(::google::protobuf::RpcController* controller, } case CheckKVRequest::CREATE_PARTITION_AFTER_FE_COMMIT: { check_create_table( - instance_id, txn_kv_, request, response, &code, &msg, + instance_id, txn_kv_, request, response, &code, &msg, stats, [](const CheckKVRequest* request) { return std::make_tuple( request->check_keys().partition_ids(), "partition", diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 33cdc5ed96a9a2..4c8f9c8ca93f1c 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -31,6 +31,7 @@ #include "common/encryption_util.h" #include "common/logging.h" #include "common/network_util.h" +#include "common/stats.h" #include "common/string_util.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" @@ -1970,7 +1971,7 @@ void MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller if (request->op() == AlterInstanceRequest::REFRESH) return; auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_] { - notify_refresh_instance(txn_kv, instance_id); + notify_refresh_instance(txn_kv, instance_id, nullptr); }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { @@ -2445,9 +2446,10 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, if (code != MetaServiceCode::OK) return; - auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_] { - notify_refresh_instance(txn_kv, instance_id); - }); + auto f = new std::function( + [instance_id = request->instance_id(), txn_kv = txn_kv_, &stats] { + notify_refresh_instance(txn_kv, instance_id, &stats); + }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { LOG(WARNING) << "notify refresh instance inplace, instance_id=" << request->instance_id(); @@ -3866,8 +3868,8 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro RPC_RATE_LIMIT(get_cluster_status) - auto get_clusters_info = [this, &request, &response, - &has_filter](const std::string& instance_id) { + auto get_clusters_info = [this, &request, &response, &has_filter, + &stats](const std::string& instance_id) { InstanceKeyInfo key_info {instance_id}; std::string key; std::string val; @@ -3879,6 +3881,11 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro LOG(WARNING) << "failed to create txn err=" << err; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); + }); err = txn->get(key, &val); LOG(INFO) << "get instance_key=" << hex(key); @@ -3923,7 +3930,8 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro msg = proto_to_json(*response); } -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id) { +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, + KVStats* stats) { LOG(INFO) << "begin notify_refresh_instance"; std::unique_ptr txn; TxnErrorCode err = txn_kv->create_txn(&txn); @@ -3931,6 +3939,13 @@ void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& i LOG(WARNING) << "failed to create txn err=" << err; return; } + std::unique_ptr> defer_status((int*)0x01, [&](int*) { + if (stats) { + stats->get_counter << txn->num_get_keys(); + stats->put_counter << txn->num_put_keys(); + stats->del_counter << txn->num_del_keys(); + } + }); std::string key = system_meta_service_registry_key(); std::string val; err = txn->get(key, &val); diff --git a/cloud/src/meta-service/meta_service_tablet_stats.cpp b/cloud/src/meta-service/meta_service_tablet_stats.cpp index e9229c031ed354..da556b6c574557 100644 --- a/cloud/src/meta-service/meta_service_tablet_stats.cpp +++ b/cloud/src/meta-service/meta_service_tablet_stats.cpp @@ -27,7 +27,6 @@ #include #include "common/logging.h" -#include "common/stats.h" #include "common/util.h" #include "meta-service/keys.h" #include "meta-service/meta_service.h" @@ -171,11 +170,10 @@ void merge_tablet_stats(TabletStatsPB& stats, const TabletStats& detached_stats) void internal_get_tablet_stats(MetaServiceCode& code, std::string& msg, Transaction* txn, const std::string& instance_id, const TabletIndexPB& idx, - TabletStatsPB& tablet_stats, bool snapshot) { + TabletStatsPB& stats, bool snapshot) { TabletStats detached_stats; - internal_get_tablet_stats(code, msg, txn, instance_id, idx, tablet_stats, detached_stats, - snapshot); - merge_tablet_stats(tablet_stats, detached_stats); + internal_get_tablet_stats(code, msg, txn, instance_id, idx, stats, detached_stats, snapshot); + merge_tablet_stats(stats, detached_stats); } MetaServiceResponseStatus parse_fix_tablet_stats_param( diff --git a/cloud/src/meta-service/meta_service_tablet_stats.h b/cloud/src/meta-service/meta_service_tablet_stats.h index 61d1cdc8e0c7b3..30cbaf56f6dc80 100644 --- a/cloud/src/meta-service/meta_service_tablet_stats.h +++ b/cloud/src/meta-service/meta_service_tablet_stats.h @@ -19,7 +19,6 @@ #include -#include "common/stats.h" #include "resource-manager/resource_manager.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index f42ddfc5d8effa..59cab27bb23309 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -23,6 +23,7 @@ #include "common/config.h" #include "common/logging.h" +#include "common/stats.h" #include "cpp/sync_point.h" #include "meta-service/doris_txn.h" #include "meta-service/keys.h" @@ -149,6 +150,9 @@ void MetaServiceImpl::begin_txn(::google::protobuf::RpcController* controller, msg = ss.str(); return; } + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); //2. Get txn id from version stamp txn.reset(); @@ -765,7 +769,8 @@ void MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr void scan_tmp_rowset( const std::string& instance_id, int64_t txn_id, std::shared_ptr txn_kv, MetaServiceCode& code, std::string& msg, int64_t* db_id, - std::vector>* tmp_rowsets_meta) { + std::vector>* tmp_rowsets_meta, + KVStats* stats) { // Create a readonly txn for scan tmp rowset std::stringstream ss; std::unique_ptr txn; @@ -777,6 +782,13 @@ void scan_tmp_rowset( LOG(WARNING) << msg; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + if (stats) { + stats->get_counter << txn->num_get_keys(); + stats->put_counter << txn->num_put_keys(); + stats->del_counter << txn->num_del_keys(); + } + }); // Get db id with txn id std::string index_val; @@ -999,7 +1011,7 @@ void commit_txn_immediately( std::shared_ptr& txn_kv, std::shared_ptr& txn_lazy_committer, MetaServiceCode& code, std::string& msg, const std::string& instance_id, int64_t db_id, std::vector>& tmp_rowsets_meta, - TxnErrorCode& err) { + TxnErrorCode& err, KVStats& stats) { std::stringstream ss; int64_t txn_id = request->txn_id(); do { @@ -1014,6 +1026,11 @@ void commit_txn_immediately( LOG(WARNING) << msg; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); + }); // Get txn info with db_id and txn_id std::string info_val; // Will be reused when saving updated txn @@ -1567,7 +1584,8 @@ void commit_txn_eventually( const CommitTxnRequest* request, CommitTxnResponse* response, std::shared_ptr& txn_kv, std::shared_ptr& txn_lazy_committer, MetaServiceCode& code, std::string& msg, const std::string& instance_id, int64_t db_id, - const std::vector>& tmp_rowsets_meta) { + const std::vector>& tmp_rowsets_meta, + KVStats& stats) { StopWatch sw; std::unique_ptr> defer_status((int*)0x01, [&](int*) { if (config::use_detailed_metrics && !instance_id.empty()) { @@ -1591,6 +1609,11 @@ void commit_txn_eventually( LOG(WARNING) << msg; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); + }); // tablet_id -> {table/index/partition}_id std::unordered_map tablet_ids; @@ -1939,7 +1962,7 @@ void commit_txn_eventually( */ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* response, std::shared_ptr& txn_kv, MetaServiceCode& code, - std::string& msg, const std::string& instance_id) { + std::string& msg, const std::string& instance_id, KVStats& stats) { std::stringstream ss; int64_t txn_id = request->txn_id(); auto sub_txn_infos = request->sub_txn_infos(); @@ -1953,6 +1976,11 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* LOG(WARNING) << msg; return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); + }); // Get db id with txn id std::string index_val; @@ -2552,13 +2580,13 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, RPC_RATE_LIMIT(commit_txn) if (request->has_is_txn_load() && request->is_txn_load()) { - commit_txn_with_sub_txn(request, response, txn_kv_, code, msg, instance_id); + commit_txn_with_sub_txn(request, response, txn_kv_, code, msg, instance_id, stats); return; } int64_t db_id; std::vector> tmp_rowsets_meta; - scan_tmp_rowset(instance_id, txn_id, txn_kv_, code, msg, &db_id, &tmp_rowsets_meta); + scan_tmp_rowset(instance_id, txn_id, txn_kv_, code, msg, &db_id, &tmp_rowsets_meta, &stats); if (code != MetaServiceCode::OK) { LOG(WARNING) << "scan_tmp_rowset failed, txn_id=" << txn_id << " code=" << code; return; @@ -2577,7 +2605,7 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, } commit_txn_immediately(request, response, txn_kv_, txn_lazy_committer_, code, msg, - instance_id, db_id, tmp_rowsets_meta, err); + instance_id, db_id, tmp_rowsets_meta, err, stats); if (MetaServiceCode::OK == code) { return; @@ -2607,7 +2635,7 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, code = MetaServiceCode::OK; msg.clear(); commit_txn_eventually(request, response, txn_kv_, txn_lazy_committer_, code, msg, instance_id, - db_id, tmp_rowsets_meta); + db_id, tmp_rowsets_meta, stats); } static void _abort_txn(const std::string& instance_id, const AbortTxnRequest* request, @@ -3117,6 +3145,8 @@ void MetaServiceImpl::begin_sub_txn(::google::protobuf::RpcController* controlle msg = ss.str(); return; } + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); // 2. Get sub txn id from version stamp txn.reset(); @@ -3607,7 +3637,7 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont * @return TxnErrorCode */ TxnErrorCode internal_clean_label(std::shared_ptr txn_kv, const std::string_view instance_id, - int64_t db_id, const std::string_view label_key) { + int64_t db_id, const std::string_view label_key, KVStats& stats) { std::string label_val; TxnLabelPB label_pb; @@ -3623,6 +3653,11 @@ TxnErrorCode internal_clean_label(std::shared_ptr txn_kv, const std::stri << " label_key=" << hex(label_key); return err; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); + }); err = txn->get(label_key, &label_val); if (err != TxnErrorCode::TXN_OK && err != TxnErrorCode::TXN_KEY_NOT_FOUND) { @@ -3778,6 +3813,11 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control << " end=" << hex(end_label_key); return; } + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + stats.get_counter << txn->num_get_keys(); + stats.put_counter << txn->num_put_keys(); + stats.del_counter << txn->num_del_keys(); + }); err = txn->get(begin_label_key, end_label_key, &it, snapshot, limit); if (err != TxnErrorCode::TXN_OK) { @@ -3799,7 +3839,7 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control begin_label_key = k; LOG(INFO) << "iterator has no more kvs. key=" << hex(k); } - err = internal_clean_label(txn_kv_, instance_id, db_id, k); + err = internal_clean_label(txn_kv_, instance_id, db_id, k, stats); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); msg = fmt::format("failed to clean txn label. err={}", err); @@ -3812,7 +3852,7 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control } else { const std::string& label = request->labels(0); const std::string label_key = txn_label_key({instance_id, db_id, label}); - TxnErrorCode err = internal_clean_label(txn_kv_, instance_id, db_id, label_key); + TxnErrorCode err = internal_clean_label(txn_kv_, instance_id, db_id, label_key, stats); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); msg = fmt::format("failed to clean txn label. err={}", err); diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp index 2260ea1ff23b7b..99aea5fed56fff 100644 --- a/cloud/src/meta-service/txn_lazy_committer.cpp +++ b/cloud/src/meta-service/txn_lazy_committer.cpp @@ -20,6 +20,7 @@ #include #include "common/logging.h" +#include "common/stats.h" #include "common/util.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" @@ -33,7 +34,8 @@ namespace doris::cloud { void scan_tmp_rowset( const std::string& instance_id, int64_t txn_id, std::shared_ptr txn_kv, MetaServiceCode& code, std::string& msg, int64_t* db_id, - std::vector>* tmp_rowsets_meta); + std::vector>* tmp_rowsets_meta, + KVStats* stats); void update_tablet_stats(const StatsTabletKeyInfo& info, const TabletStats& stats, std::unique_ptr& txn, MetaServiceCode& code, @@ -308,7 +310,7 @@ void TxnLazyCommitTask::commit() { int64_t db_id; std::vector> all_tmp_rowset_metas; scan_tmp_rowset(instance_id_, txn_id_, txn_kv_, code_, msg_, &db_id, - &all_tmp_rowset_metas); + &all_tmp_rowset_metas, nullptr); if (code_ != MetaServiceCode::OK) { LOG(WARNING) << "scan_tmp_rowset failed, txn_id=" << txn_id_ << " code=" << code_; break; diff --git a/cloud/test/meta_server_test.cpp b/cloud/test/meta_server_test.cpp index 15fd50785ac54d..dcfd28285c317a 100644 --- a/cloud/test/meta_server_test.cpp +++ b/cloud/test/meta_server_test.cpp @@ -34,6 +34,7 @@ #include "common/config.h" #include "common/logging.h" +#include "common/stats.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" #include "meta-service/mem_txn_kv.h" @@ -62,7 +63,8 @@ int main(int argc, char** argv) { } namespace doris::cloud { -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id); +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, + KVStats* stats); } // namespace doris::cloud TEST(MetaServerTest, FQDNRefreshInstance) { @@ -136,7 +138,7 @@ TEST(MetaServerTest, FQDNRefreshInstance) { // Refresh instance with FQDN endpoint. config::hostname = ""; - notify_refresh_instance(txn_kv, "fqdn_instance_id"); + notify_refresh_instance(txn_kv, "fqdn_instance_id", nullptr); bool refreshed = false; for (size_t i = 0; i < 100; ++i) { diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index a8c6e4bdb09002..642c7bce54327f 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -91,7 +91,7 @@ void start_compaction_job(MetaService* meta_service, int64_t tablet_id, const st meta_service->start_tablet_job(&cntl, &req, &res, nullptr); }; -std::unique_ptr get_meta_service(bool mock_resource_mgr) { +std::unique_ptr get_meta_service() { int ret = 0; // MemKv auto txn_kv = std::dynamic_pointer_cast(std::make_shared()); @@ -112,10 +112,6 @@ std::unique_ptr get_meta_service(bool mock_resource_mgr) { return std::make_unique(std::move(meta_service)); } -std::unique_ptr get_meta_service() { - return get_meta_service(true); -} - std::unique_ptr get_fdb_meta_service() { config::fdb_cluster_file_path = "fdb.cluster"; static auto txn_kv = std::dynamic_pointer_cast(std::make_shared()); @@ -207,6 +203,32 @@ static void create_tablet_with_db_id(MetaServiceProxy* meta_service, int64_t db_ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; } +void create_tablet(MetaService* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, bool enable_mow, + bool not_ready = false) { + brpc::Controller cntl; + CreateTabletsRequest req; + CreateTabletsResponse res; + auto* tablet = req.add_tablet_metas(); + tablet->set_tablet_state(not_ready ? doris::TabletStatePB::PB_NOTREADY + : doris::TabletStatePB::PB_RUNNING); + tablet->set_table_id(table_id); + tablet->set_index_id(index_id); + tablet->set_partition_id(partition_id); + tablet->set_tablet_id(tablet_id); + tablet->set_enable_unique_key_merge_on_write(enable_mow); + auto* schema = tablet->mutable_schema(); + schema->set_schema_version(0); + auto* first_rowset = tablet->add_rs_metas(); + first_rowset->set_rowset_id(0); // required + first_rowset->set_rowset_id_v2(next_rowset_id()); + first_rowset->set_start_version(0); + first_rowset->set_end_version(1); + first_rowset->mutable_tablet_schema()->CopyFrom(*schema); + meta_service->create_tablets(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; +} + static void create_tablet(MetaServiceProxy* meta_service, int64_t table_id, int64_t index_id, int64_t partition_id, int64_t tablet_id, const std::string& rowset_id, int32_t schema_version) { @@ -267,6 +289,28 @@ static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id, return rowset; } +static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id, int index_id, + int partition_id, int64_t version = -1, + int num_rows = 100) { + doris::RowsetMetaCloudPB rowset; + rowset.set_rowset_id(0); // required + rowset.set_rowset_id_v2(next_rowset_id()); + rowset.set_tablet_id(tablet_id); + rowset.set_partition_id(partition_id); + rowset.set_index_id(index_id); + rowset.set_txn_id(txn_id); + if (version > 0) { + rowset.set_start_version(version); + rowset.set_end_version(version); + } + rowset.set_num_segments(0); + rowset.set_num_rows(0); + rowset.set_data_disk_size(0); + rowset.mutable_tablet_schema()->set_schema_version(0); + rowset.set_txn_expiration(::time(nullptr)); // Required by DCHECK + return rowset; +} + static void check_get_tablet(MetaServiceProxy* meta_service, int64_t tablet_id, int32_t schema_version) { brpc::Controller cntl; @@ -280,6 +324,16 @@ static void check_get_tablet(MetaServiceProxy* meta_service, int64_t tablet_id, EXPECT_EQ(res.tablet_meta().schema_version(), schema_version) << tablet_id; }; +static void create_and_commit_rowset(MetaServiceProxy* meta_service, int64_t table_id, + int64_t index_id, int64_t partition_id, int64_t tablet_id, + int64_t txn_id) { + create_tablet(meta_service, table_id, index_id, partition_id, tablet_id); + auto tmp_rowset = create_rowset(txn_id, tablet_id, partition_id); + CreateRowsetResponse res; + commit_rowset(meta_service, tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); +} + static void prepare_rowset(MetaServiceProxy* meta_service, const doris::RowsetMetaCloudPB& rowset, CreateRowsetResponse& res) { brpc::Controller cntl; @@ -402,19 +456,77 @@ static void update_delete_bitmap(MetaServiceProxy* meta_service, nullptr); } +void start_schema_change_job(MetaServiceProxy* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, int64_t new_tablet_id, + const std::string& job_id, const std::string& initiator, + StartTabletJobResponse& res, int64_t alter_version = -1) { + brpc::Controller cntl; + StartTabletJobRequest req; + req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id); + auto* sc = req.mutable_job()->mutable_schema_change(); + sc->set_id(job_id); + sc->set_initiator(initiator); + sc->mutable_new_tablet_idx()->set_tablet_id(new_tablet_id); + if (alter_version != -1) { + sc->set_alter_version(alter_version); + } + long now = time(nullptr); + sc->set_expiration(now + 12); + meta_service->start_tablet_job(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) + << job_id << ' ' << initiator << ' ' << res.status().msg(); +}; + +void finish_schema_change_job( + MetaService* meta_service, int64_t tablet_id, int64_t new_tablet_id, + const std::string& job_id, const std::string& initiator, + const std::vector& output_rowsets, FinishTabletJobResponse& res, + FinishTabletJobRequest_Action action = FinishTabletJobRequest::COMMIT) { + brpc::Controller cntl; + FinishTabletJobRequest req; + req.set_action(action); + req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id); + auto* sc = req.mutable_job()->mutable_schema_change(); + sc->mutable_new_tablet_idx()->set_tablet_id(new_tablet_id); + if (output_rowsets.empty()) { + sc->set_alter_version(0); + } else { + sc->set_alter_version(output_rowsets.back().end_version()); + for (const auto& rowset : output_rowsets) { + sc->add_txn_ids(rowset.txn_id()); + sc->add_output_versions(rowset.end_version()); + sc->set_num_output_rows(sc->num_output_rows() + rowset.num_rows()); + sc->set_num_output_segments(sc->num_output_segments() + rowset.num_segments()); + sc->set_size_output_rowsets(sc->size_output_rowsets() + rowset.total_disk_size()); + sc->set_index_size_output_rowsets(sc->index_size_output_rowsets() + + rowset.index_disk_size()); + sc->set_segment_size_output_rowsets(sc->segment_size_output_rowsets() + + rowset.data_disk_size()); + } + sc->set_num_output_rowsets(output_rowsets.size()); + } + sc->set_id(job_id); + sc->set_initiator(initiator); + sc->set_delete_bitmap_lock_initiator(12345); + meta_service->finish_tablet_job(&cntl, &req, &res, nullptr); +} + // create_tablets TEST(RpcKvBvarTest, DISABLED_CreateTablets) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); - LOG(INFO) << "CreateTablets: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_create_tablets_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_create_tablets_write_counter.get_value()); + LOG(INFO) << "CreateTablets: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_tablets_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_tablets_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_create_tablets_del_counter.get_value()); } // get_tablet @@ -425,8 +537,9 @@ TEST(RpcKvBvarTest, DISABLED_GetTablet) { constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; brpc::Controller cntl; GetTabletRequest req; @@ -435,9 +548,11 @@ TEST(RpcKvBvarTest, DISABLED_GetTablet) { GetTabletResponse resp; meta_service->get_tablet(&cntl, &req, &resp, nullptr); - LOG(INFO) << "GetTablet: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_tablet_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_tablet_write_counter.get_value()); + LOG(INFO) << "GetTablet: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_tablet_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_tablet_del_counter.get_value()); } // get_tablet_stats @@ -448,15 +563,18 @@ TEST(RpcKvBvarTest, DISABLED_GetTabletStats) { constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; GetTabletStatsResponse res; get_tablet_stats(meta_service.get(), table_id, index_id, partition_id, tablet_id, res); - LOG(INFO) << "GetTabletStats: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_tablet_stats_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_tablet_stats_write_counter.get_value()); + LOG(INFO) << "GetTabletStats: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_stats_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_tablet_stats_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_tablet_stats_del_counter.get_value()); } // update_tablet @@ -474,14 +592,17 @@ TEST(RpcKvBvarTest, DISABLED_UpdateTablet) { tablet_meta_info->set_tablet_id(tablet_id); tablet_meta_info->set_is_in_memory(true); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; meta_service->update_tablet(&cntl, &req, &resp, nullptr); - LOG(INFO) << "UpdateTablet: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_update_tablet_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_update_tablet_write_counter.get_value()); + LOG(INFO) << "UpdateTablet: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tablet_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tablet_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_update_tablet_del_counter.get_value()); } // update_tablet_schema @@ -498,19 +619,226 @@ TEST(RpcKvBvarTest, DISABLED_UpdateTablet) { // req.set_tablet_id(tablet_id); // req.set_cloud_unique_id("test_cloud_unique_id"); -// mem_kv->read_count_ = 0; -// mem_kv->write_count_ = 0; +// mem_kv->get_count_ = 0; +// mem_kv->put_count_ = 0; +// mem_kv->del_count_ = 0; // meta_service->update_tablet_schema(&cntl, &req, &resp, nullptr); -// LOG(INFO) << "UpdateTabletSchema: " << mem_kv->read_count_ << ", " -// << mem_kv->write_count_; -// ASSERT_EQ(mem_kv->read_count_, -// g_bvar_rpc_kv_update_tablet_schema_read_counter.get_value()); -// ASSERT_EQ(mem_kv->write_count_, -// g_bvar_rpc_kv_update_tablet_schema_write_counter.get_value()); +// LOG(INFO) << "UpdateTabletSchema: " << mem_kv->get_count_ << ", " +// << mem_kv->put_count_ << ", " << mem_kv->del_count_; +// ASSERT_EQ(mem_kv->get_count_, +// g_bvar_rpc_kv_update_tablet_schema_get_counter.get_value()); +// ASSERT_EQ(mem_kv->put_count_, +// g_bvar_rpc_kv_update_tablet_schema_put_counter.get_value()); +// ASSERT_EQ(mem_kv->del_count_, +// g_bvar_rpc_kv_update_tablet_schema_del_counter.get_value()); // } +// begin_txn +TEST(RpcKvBvarTest, DISABLED_BeginTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + int64_t txn_id = 0; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + + LOG(INFO) << "BeginTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_txn_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_txn_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_begin_txn_del_counter.get_value()); +} + +// commit_txn +TEST(RpcKvBvarTest, DISABLED_CommitTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + int64_t txn_id = 0; + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + commit_txn(meta_service.get(), db_id, txn_id, label); + + LOG(INFO) << "CommitTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_txn_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_txn_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_txn_del_counter.get_value()); +} + +// precommit_txn +TEST(RpcKvBvarTest, DISABLED_PrecommitTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + const int64_t db_id = 563413; + const int64_t table_id = 417417878; + const std::string& label = "label_123dae121das"; + int64_t txn_id = -1; + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info; + txn_info.set_db_id(db_id); + txn_info.set_label(label); + txn_info.add_table_ids(table_id); + txn_info.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id = res.txn_id(); + ASSERT_GT(txn_id, -1); + } + + std::unique_ptr txn; + TxnErrorCode err = meta_service->txn_kv()->create_txn(&txn); + ASSERT_EQ(err, TxnErrorCode::TXN_OK); + + const std::string info_key = txn_info_key({mock_instance, db_id, txn_id}); + std::string info_val; + ASSERT_EQ(txn->get(info_key, &info_val), TxnErrorCode::TXN_OK); + TxnInfoPB txn_info; + txn_info.ParseFromString(info_val); + ASSERT_EQ(txn_info.status(), TxnStatusPB::TXN_STATUS_PREPARED); + + brpc::Controller cntl; + PrecommitTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_precommit_timeout_ms(36000); + PrecommitTxnResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->precommit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "PrecommitTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_precommit_txn_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_precommit_txn_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_precommit_txn_del_counter.get_value()); +} + +// abort_txn +TEST(RpcKvBvarTest, DISABLED_AbortTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + int64_t txn_id = 0; + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + brpc::Controller cntl; + AbortTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_reason("test"); + AbortTxnResponse res; + meta_service->abort_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "AbortTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_txn_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_txn_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_abort_txn_del_counter.get_value()); +} + +// get_txn +TEST(RpcKvBvarTest, DISABLED_GetTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + int64_t txn_id = 0; + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + brpc::Controller cntl; + GetTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_db_id(db_id); + GetTxnResponse res; + meta_service->get_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "GetTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_txn_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_txn_del_counter.get_value()); +} + +// get_txn_id +TEST(RpcKvBvarTest, DISABLED_GetTxnId) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + int64_t txn_id = 0; + + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + + brpc::Controller cntl; + GetTxnIdRequest req; + GetTxnIdResponse res; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_label(label); + req.set_db_id(db_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_txn_id(&cntl, &req, &res, nullptr); + + LOG(INFO) << "GetTxnId: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_id_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_txn_id_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_txn_id_del_counter.get_value()); +} + // prepare_rowset TEST(RpcKvBvarTest, DISABLED_PrepareRowset) { int64_t db_id = 100201; @@ -527,14 +855,17 @@ TEST(RpcKvBvarTest, DISABLED_PrepareRowset) { rowset.mutable_load_id()->set_hi(123); rowset.mutable_load_id()->set_lo(456); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; prepare_rowset(meta_service.get(), rowset, res); - LOG(INFO) << "PrepareRowset: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_prepare_rowset_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_prepare_rowset_write_counter.get_value()); + LOG(INFO) << "PrepareRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_rowset_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_rowset_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_prepare_rowset_del_counter.get_value()); } // get_rowset @@ -549,16 +880,19 @@ TEST(RpcKvBvarTest, DISABLED_GetRowset) { // check get tablet response check_get_tablet(meta_service.get(), tablet_id, 1); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; // check get rowset response GetRowsetResponse get_rowset_res; get_rowset(meta_service.get(), table_id, index_id, partition_id, tablet_id, get_rowset_res); - LOG(INFO) << "GetRowset: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_rowset_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_rowset_write_counter.get_value()); + LOG(INFO) << "GetRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_rowset_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_rowset_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_rowset_del_counter.get_value()); } // update_tmp_rowset @@ -589,8 +923,9 @@ TEST(RpcKvBvarTest, DISABLED_UpdateTmpRowset) { EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); req->mutable_rowset_meta()->CopyFrom(rowset); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; meta_service->update_tmp_rowset(&cntl, req, &res, nullptr); @@ -598,9 +933,11 @@ TEST(RpcKvBvarTest, DISABLED_UpdateTmpRowset) { delete req; } - LOG(INFO) << "UpdateTmpRowset: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_update_tmp_rowset_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_update_tmp_rowset_write_counter.get_value()); + LOG(INFO) << "UpdateTmpRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tmp_rowset_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tmp_rowset_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_update_tmp_rowset_del_counter.get_value()); } // commit_rowset @@ -613,15 +950,18 @@ TEST(RpcKvBvarTest, DISABLED_CommitRowset) { create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); auto tmp_rowset = create_rowset(txn_id, tablet_id, partition_id); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; CreateRowsetResponse res; commit_rowset(meta_service.get(), tmp_rowset, res); - LOG(INFO) << "CommitRowset: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_commit_rowset_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_commit_rowset_write_counter.get_value()); + LOG(INFO) << "CommitRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_rowset_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_rowset_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_rowset_del_counter.get_value()); } // get_version @@ -640,15 +980,18 @@ TEST(RpcKvBvarTest, DISABLED_GetVersion) { req.set_table_id(table_id); req.set_partition_id(partition_id); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; GetVersionResponse resp; meta_service->get_version(&ctrl, &req, &resp, nullptr); - LOG(INFO) << "GetVersion: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_version_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_version_write_counter.get_value()); + LOG(INFO) << "GetVersion: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_version_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_version_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_version_del_counter.get_value()); } // get_schema_dict @@ -670,15 +1013,18 @@ TEST(RpcKvBvarTest, DISABLED_GetSchemaDict) { txn->put(dict_key, "dict_val"); EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; GetSchemaDictResponse resp; meta_service->get_schema_dict(&ctrl, &req, &resp, nullptr); - LOG(INFO) << "GetSchemaDict: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_schema_dict_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_schema_dict_write_counter.get_value()); + LOG(INFO) << "GetSchemaDict: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_schema_dict_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_schema_dict_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_schema_dict_del_counter.get_value()); } // get_delete_bitmap_update_lock @@ -699,17 +1045,20 @@ TEST(RpcKvBvarTest, DISABLED_GetDeleteBitmapUpdateLock) { int64_t lock_id = -2; int64_t initiator = 1009; - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); - LOG(INFO) << "GetDeleteBitmapUpdateLock: " << mem_kv->read_count_ << ", " - << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, - g_bvar_rpc_kv_get_delete_bitmap_update_lock_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, - g_bvar_rpc_kv_get_delete_bitmap_update_lock_write_counter.get_value()); + LOG(INFO) << "GetDeleteBitmapUpdateLock: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ + << ", " << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter.get_value()); } // update_delete_bitmap @@ -737,15 +1086,18 @@ TEST(RpcKvBvarTest, DISABLED_UpdateDeleteBitmap) { // will be splited and stored in 5 KVs std::string data1(split_size * 5, 'c'); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; update_delete_bitmap(meta_service.get(), update_delete_bitmap_req, update_delete_bitmap_res, table_id, t1p1, lock_id, initiator, tablet_id, txn_id, version, data1); - LOG(INFO) << "UpdateDeleteBitmap: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_update_delete_bitmap_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_update_delete_bitmap_write_counter.get_value()); + LOG(INFO) << "UpdateDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_delete_bitmap_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_delete_bitmap_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_update_delete_bitmap_del_counter.get_value()); } // get_delete_bitmap @@ -783,15 +1135,18 @@ TEST(RpcKvBvarTest, DISABLED_GetDeleteBitmap) { get_delete_bitmap_req.add_begin_versions(0); get_delete_bitmap_req.add_end_versions(version); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; meta_service->get_delete_bitmap(reinterpret_cast(&ctrl), &get_delete_bitmap_req, &get_delete_bitmap_res, nullptr); - LOG(INFO) << "GetDeleteBitmap: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_get_delete_bitmap_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_get_delete_bitmap_write_counter.get_value()); + LOG(INFO) << "GetDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_delete_bitmap_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_delete_bitmap_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_delete_bitmap_del_counter.get_value()); } // remove_delete_bitmap_update_lock @@ -822,19 +1177,22 @@ TEST(RpcKvBvarTest, DISABLED_RemoveDeleteBitmapUpdateLock) { remove_req.set_lock_id(lock_id); remove_req.set_initiator(initiator); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; meta_service->remove_delete_bitmap_update_lock( reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &remove_req, &remove_res, nullptr); - LOG(INFO) << "RemoveDeleteBitmapUpdateLock: " << mem_kv->read_count_ << ", " - << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, - g_bvar_rpc_kv_remove_delete_bitmap_update_lock_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, - g_bvar_rpc_kv_remove_delete_bitmap_update_lock_write_counter.get_value()); + LOG(INFO) << "RemoveDeleteBitmapUpdateLock: " << mem_kv->get_count_ << ", " + << mem_kv->put_count_ << ", " << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter.get_value()); } // remove_delete_bitmap @@ -871,14 +1229,17 @@ TEST(RpcKvBvarTest, DISABLED_RemoveDeleteBitmap) { req.add_rowset_ids("rowset_ids"); req.set_cloud_unique_id("test_cloud_unique_id"); - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; meta_service->remove_delete_bitmap(&ctrl, &req, &resp, nullptr); - LOG(INFO) << "RemoveDeleteBitmap: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_remove_delete_bitmap_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_remove_delete_bitmap_write_counter.get_value()); + LOG(INFO) << "RemoveDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_remove_delete_bitmap_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_remove_delete_bitmap_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_remove_delete_bitmap_del_counter.get_value()); } // start_tablet_job @@ -892,58 +1253,1976 @@ TEST(RpcKvBvarTest, DISABLED_StartTabletJob) { create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); StartTabletJobResponse res; - mem_kv->read_count_ = 0; - mem_kv->write_count_ = 0; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, TabletCompactionJobPB::BASE, res); - LOG(INFO) << "StartTabletJob: " << mem_kv->read_count_ << ", " << mem_kv->write_count_; - ASSERT_EQ(mem_kv->read_count_, g_bvar_rpc_kv_start_tablet_job_read_counter.get_value()); - ASSERT_EQ(mem_kv->write_count_, g_bvar_rpc_kv_start_tablet_job_write_counter.get_value()); + LOG(INFO) << "StartTabletJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_start_tablet_job_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_start_tablet_job_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_start_tablet_job_del_counter.get_value()); } // finish_tablet_job +TEST(RpcKvBvarTest, DISABLED_FinishTabletJob) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + + int64_t table_id = 1; + int64_t index_id = 2; + int64_t partition_id = 3; + int64_t tablet_id = 4; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id, false)); + + StartTabletJobResponse res; + start_compaction_job(meta_service.get(), tablet_id, "job1", "BE1", 0, 0, + TabletCompactionJobPB::CUMULATIVE, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + res.Clear(); + + int64_t new_tablet_id = 11; + ASSERT_NO_FATAL_FAILURE(create_tablet(meta_service.get(), table_id, index_id, partition_id, + new_tablet_id, false, true)); + StartTabletJobResponse sc_res; + ASSERT_NO_FATAL_FAILURE(start_schema_change_job(meta_service.get(), table_id, index_id, + partition_id, tablet_id, new_tablet_id, "job2", + "BE1", sc_res)); + + long now = time(nullptr); + FinishTabletJobRequest req; + FinishTabletJobResponse finish_res_2; + req.set_action(FinishTabletJobRequest::LEASE); + auto* compaction = req.mutable_job()->add_compaction(); + compaction->set_id("job1"); + compaction->set_initiator("BE1"); + compaction->set_lease(now + 10); + req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->finish_tablet_job(&cntl, &req, &finish_res_2, nullptr); + + LOG(INFO) << "FinishTabletJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_finish_tablet_job_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_finish_tablet_job_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_finish_tablet_job_del_counter.get_value()); +} // prepare_index +TEST(RpcKvBvarTest, DISABLED_PrepareIndex) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_cloud_instance_id"; + + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + constexpr int64_t tablet_id = 10004; + + std::unique_ptr txn; + doris::TabletMetaCloudPB tablet_pb; + tablet_pb.set_table_id(table_id); + tablet_pb.set_index_id(index_id); + tablet_pb.set_partition_id(partition_id); + tablet_pb.set_tablet_id(tablet_id); + auto tablet_key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id}); + auto tablet_val = tablet_pb.SerializeAsString(); + RecycleIndexPB index_pb; + auto index_key = recycle_index_key({instance_id, index_id}); + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + std::string val; + + brpc::Controller ctrl; + IndexRequest req; + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + req.set_is_new_table(true); + IndexResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->prepare_index(&ctrl, &req, &res, nullptr); + + LOG(INFO) << "PrepareIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_index_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_index_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_prepare_index_del_counter.get_value()); +} + // commit_index +TEST(RpcKvBvarTest, DISABLED_CommitIndex) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_cloud_instance_id"; + + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + constexpr int64_t tablet_id = 10004; + + std::unique_ptr txn; + doris::TabletMetaCloudPB tablet_pb; + tablet_pb.set_table_id(table_id); + tablet_pb.set_index_id(index_id); + tablet_pb.set_partition_id(partition_id); + tablet_pb.set_tablet_id(tablet_id); + auto tablet_key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id}); + auto tablet_val = tablet_pb.SerializeAsString(); + RecycleIndexPB index_pb; + auto index_key = recycle_index_key({instance_id, index_id}); + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + std::string val; + + brpc::Controller ctrl; + IndexRequest req; + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + req.set_is_new_table(true); + IndexResponse res; + meta_service->prepare_index(&ctrl, &req, &res, nullptr); + res.Clear(); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->commit_index(&ctrl, &req, &res, nullptr); + + LOG(INFO) << "CommitIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_index_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_index_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_index_del_counter.get_value()); +} + // drop_index +TEST(RpcKvBvarTest, DISABLED_DropIndex) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 4524364; + int64_t table_id = 65354; + int64_t index_id = 658432; + int64_t partition_id = 76553; + std::string mock_instance = "test_instance"; + const std::string label = "test_label_67a34e2q1231"; + + int64_t tablet_id_base = 2313324; + for (int i = 0; i < 10; ++i) { + create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, partition_id, + tablet_id_base + i); + } + int txn_id {}; + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id = res.txn_id(); + ASSERT_GT(txn_id, 0); + } + { + for (int i = 0; i < 10; ++i) { + auto tmp_rowset = + create_rowset(txn_id, tablet_id_base + i, index_id, partition_id, -1, 100); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + } + + brpc::Controller cntl; + IndexRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + IndexResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->drop_index(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "DropIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_index_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_index_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_drop_index_del_counter.get_value()); +} + // prepare_partition +TEST(RpcKvBvarTest, DISABLED_PreparePartition) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_cloud_instance_id"; + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + constexpr int64_t tablet_id = 10004; + + std::unique_ptr txn; + doris::TabletMetaCloudPB tablet_pb; + tablet_pb.set_table_id(table_id); + tablet_pb.set_index_id(index_id); + tablet_pb.set_partition_id(partition_id); + tablet_pb.set_tablet_id(tablet_id); + auto tablet_key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id}); + auto tablet_val = tablet_pb.SerializeAsString(); + RecyclePartitionPB partition_pb; + auto partition_key = recycle_partition_key({instance_id, partition_id}); + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + std::string val; + brpc::Controller ctrl; + PartitionRequest req; + PartitionResponse res; + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + req.add_partition_ids(partition_id); + res.Clear(); + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->atomic_add(tbl_version_key, 1); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->prepare_partition(&ctrl, &req, &res, nullptr); + + LOG(INFO) << "PreparePartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_partition_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_partition_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_prepare_partition_del_counter.get_value()); +} + // commit_partition -// drop_partition -// check_kv -// get_obj_store_info -// alter_storage_vault +TEST(RpcKvBvarTest, DISABLED_CommitPartition) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_cloud_instance_id"; + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + constexpr int64_t tablet_id = 10004; + + std::unique_ptr txn; + doris::TabletMetaCloudPB tablet_pb; + tablet_pb.set_table_id(table_id); + tablet_pb.set_index_id(index_id); + tablet_pb.set_partition_id(partition_id); + tablet_pb.set_tablet_id(tablet_id); + auto tablet_key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id}); + auto tablet_val = tablet_pb.SerializeAsString(); + RecyclePartitionPB partition_pb; + auto partition_key = recycle_partition_key({instance_id, partition_id}); + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + std::string val; + brpc::Controller ctrl; + PartitionRequest req; + PartitionResponse res; + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + req.add_partition_ids(partition_id); + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->atomic_add(tbl_version_key, 1); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + meta_service->prepare_partition(&ctrl, &req, &res, nullptr); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->commit_partition(&ctrl, &req, &res, nullptr); + + LOG(INFO) << "CommitPartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_partition_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_partition_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_partition_del_counter.get_value()); +} + +// check_kv +TEST(RpcKvBvarTest, DISABLED_CheckKv) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_instance"; + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + + std::unique_ptr txn; + RecyclePartitionPB partition_pb; + auto partition_key = recycle_partition_key({instance_id, 10004}); + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + brpc::Controller ctrl; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(partition_key, "val"); + txn->atomic_add(tbl_version_key, 1); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + CheckKVRequest req_check; + CheckKVResponse res_check; + req_check.set_op(CheckKVRequest::CREATE_PARTITION_AFTER_FE_COMMIT); + CheckKeyInfos check_keys_pb; + check_keys_pb.add_table_ids(table_id + 1); + check_keys_pb.add_index_ids(index_id + 1); + check_keys_pb.add_partition_ids(partition_id + 1); + req_check.mutable_check_keys()->CopyFrom(check_keys_pb); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->check_kv(&ctrl, &req_check, &res_check, nullptr); + + LOG(INFO) << "CheckKv: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_check_kv_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_check_kv_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_check_kv_del_counter.get_value()); +} + +// drop_partition +TEST(RpcKvBvarTest, DISABLED_DropPartition) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_instance"; + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->atomic_add(tbl_version_key, 1); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + brpc::Controller ctrl; + PartitionRequest req; + PartitionResponse res; + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + req.add_partition_ids(partition_id); + req.set_need_update_table_version(true); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->drop_partition(&ctrl, &req, &res, nullptr); + + LOG(INFO) << "DropPartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_partition_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_partition_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_drop_partition_del_counter.get_value()); +} + +// get_obj_store_info +TEST(RpcKvBvarTest, DISABLED_GetObjStoreInfo) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + auto rate_limiter = std::make_shared(); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + InstanceKeyInfo key_info {"test_instance"}; + std::string key; + instance_key(key_info, &key); + txn->put(key, "val"); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + brpc::Controller cntl; + GetObjStoreInfoResponse res; + GetObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_obj_store_info(&cntl, &req, &res, nullptr); + + LOG(INFO) << "GetObjStoreInfo: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_obj_store_info_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_obj_store_info_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_obj_store_info_del_counter.get_value()); +} + +// alter_storage_vault +TEST(RpcKvBvarTest, DISABLED_AlterStorageVault) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + constexpr char vault_name[] = "test_alter_s3_vault"; + + InstanceKeyInfo key_info {"test_instance"}; + std::string key; + instance_key(key_info, &key); + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(key, "val"); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + StorageVaultPB vault; + vault.mutable_obj_info()->set_ak("new_ak"); + vault.set_name(vault_name); + req.mutable_vault()->CopyFrom(vault); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->alter_storage_vault(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + + LOG(INFO) << "AlterStorageVault: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_storage_vault_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_storage_vault_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_alter_storage_vault_del_counter.get_value()); +} + // alter_obj_store_info +TEST(RpcKvBvarTest, DISABLED_AlterObjStoreInfo) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "selectdbselectdbselectdbselectdb"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string key; + std::string val; + InstanceKeyInfo key_info {"test_instance"}; + instance_key(key_info, &key); + + ObjectStoreInfoPB obj_info; + obj_info.set_id("1"); + obj_info.set_ak("ak"); + obj_info.set_sk("sk"); + InstanceInfoPB instance; + instance.add_obj_info()->CopyFrom(obj_info); + val = instance.SerializeAsString(); + txn->put(key, val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t"; + + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK); + req.mutable_obj()->set_id("1"); + req.mutable_obj()->set_ak("new_ak"); + req.mutable_obj()->set_sk(plain_sk); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->alter_obj_store_info(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + + LOG(INFO) << "AlterObjStoreInfo: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_obj_store_info_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_obj_store_info_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_alter_obj_store_info_del_counter.get_value()); + SyncPoint::get_instance()->disable_processing(); + SyncPoint::get_instance()->clear_all_call_backs(); +} + // update_ak_sk +TEST(RpcKvBvarTest, DISABLED_UpdateAkSk) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "selectdbselectdbselectdbselectdb"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + + std::string cipher_sk = "JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO"; + std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t"; + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string key; + std::string val; + InstanceKeyInfo key_info {"test_instance"}; + instance_key(key_info, &key); + + ObjectStoreInfoPB obj_info; + + obj_info.set_user_id("111"); + + obj_info.set_ak("ak"); + obj_info.set_sk("sk"); + InstanceInfoPB instance; + instance.add_obj_info()->CopyFrom(obj_info); + val = instance.SerializeAsString(); + txn->put(key, val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + UpdateAkSkRequest req; + req.set_instance_id("test_instance"); + RamUserPB ram_user; + ram_user.set_user_id("111"); + + ram_user.set_ak("new_ak"); + ram_user.set_sk(plain_sk); + req.add_internal_bucket_user()->CopyFrom(ram_user); + + brpc::Controller cntl; + UpdateAkSkResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->update_ak_sk(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "UpdateAkSk: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_ak_sk_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_ak_sk_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_update_ak_sk_del_counter.get_value()); + + SyncPoint::get_instance()->disable_processing(); + SyncPoint::get_instance()->clear_all_call_backs(); +} + // create_instance +TEST(RpcKvBvarTest, DISABLED_CreateInstance) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + brpc::Controller cntl; + CreateInstanceRequest req; + req.set_instance_id("test_instance"); + req.set_user_id("test_user"); + req.set_name("test_name"); + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + req.mutable_obj_info()->CopyFrom(obj); + + auto* sp = SyncPoint::get_instance(); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->enable_processing(); + CreateInstanceResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "CreateInstance: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_instance_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_instance_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_create_instance_del_counter.get_value()); + + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + // get_instance +TEST(RpcKvBvarTest, DISABLED_GetInstance) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + auto* sp = SyncPoint::get_instance(); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->enable_processing(); + brpc::Controller cntl; + { + CreateInstanceRequest req; + req.set_instance_id("test_instance"); + req.set_user_id("test_user"); + req.set_name("test_name"); + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + req.mutable_obj_info()->CopyFrom(obj); + + CreateInstanceResponse res; + + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + } + GetInstanceRequest req; + GetInstanceResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + req.set_cloud_unique_id("1:test_instance:m-n3qdpyal27rh8iprxx"); + meta_service->get_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "GetInstance: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_instance_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_instance_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_instance_del_counter.get_value()); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + // alter_cluster +// alter cluster have not do kv op +// TEST(RpcKvBvarTest, AlterCluster) { +// auto meta_service = get_meta_service(); +// auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + +// brpc::Controller cntl; +// AlterClusterRequest req; +// req.set_instance_id(mock_instance); +// req.mutable_cluster()->set_cluster_name(mock_cluster_name); +// req.set_op(AlterClusterRequest::ADD_CLUSTER); +// AlterClusterResponse res; + +// mem_kv->get_count_ = 0; +// mem_kv->put_count_ = 0; +// mem_kv->del_count_ = 0; + +// meta_service->alter_cluster(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, +// &res, nullptr); + +// LOG(INFO) << "AlterCluster: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " +// << mem_kv->del_count_; +// ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_cluster_get_counter.get_value()); +// ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_cluster_put_counter.get_value()); +// ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_alter_cluster_del_counter.get_value()); +// } + // get_cluster +TEST(RpcKvBvarTest, DISABLED_GetCluster) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + InstanceKeyInfo key_info {mock_instance}; + std::string key; + std::string val; + instance_key(key_info, &key); + + InstanceInfoPB instance; + instance.set_instance_id(mock_instance); + ClusterPB c1; + c1.set_cluster_name(mock_cluster_name); + c1.set_cluster_id(mock_cluster_id); + c1.add_mysql_user_name()->append("m1"); + instance.add_clusters()->CopyFrom(c1); + val = instance.SerializeAsString(); + + std::unique_ptr txn; + std::string get_val; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(key, val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + brpc::Controller cntl; + GetClusterRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_cluster_id(mock_cluster_id); + req.set_cluster_name("test_cluster"); + GetClusterResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_cluster(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "GetCluster: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_cluster_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_cluster_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_cluster_del_counter.get_value()); +} + // create_stage +TEST(RpcKvBvarTest, DISABLED_CreateStage) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + brpc::Controller cntl; + const auto* cloud_unique_id = "test_cloud_unique_id"; + std::string instance_id = "stage_test_instance_id"; + [[maybe_unused]] auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer( + (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->set_call_back("decrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* key = try_any_cast(args[0]); + *key = "test"; + auto* ret = try_any_cast(args[1]); + *ret = 0; + }); + sp->enable_processing(); + + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + + RamUserPB ram_user; + ram_user.set_user_id("test_user_id"); + ram_user.set_ak("test_ak"); + ram_user.set_sk("test_sk"); + EncryptionInfoPB encry_info; + encry_info.set_encryption_method("encry_method_test"); + encry_info.set_key_id(1111); + ram_user.mutable_encryption_info()->CopyFrom(encry_info); + + // create instance + { + CreateInstanceRequest req; + req.set_instance_id(instance_id); + req.set_user_id("test_user"); + req.set_name("test_name"); + req.mutable_ram_user()->CopyFrom(ram_user); + req.mutable_obj_info()->CopyFrom(obj); + + CreateInstanceResponse res; + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + // create an internal stage + CreateStageRequest create_stage_request; + StagePB stage; + stage.set_type(StagePB::INTERNAL); + stage.add_mysql_user_name("root"); + stage.add_mysql_user_id("root_id"); + stage.set_stage_id("internal_stage_id"); + create_stage_request.set_cloud_unique_id(cloud_unique_id); + create_stage_request.mutable_stage()->CopyFrom(stage); + CreateStageResponse create_stage_response; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->create_stage(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &create_stage_request, &create_stage_response, nullptr); + ASSERT_EQ(create_stage_response.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "CreateStage: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_stage_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_stage_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_create_stage_del_counter.get_value()); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + // get_stage +TEST(RpcKvBvarTest, DISABLED_GetStage) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + brpc::Controller cntl; + const auto* cloud_unique_id = "test_cloud_unique_id"; + std::string instance_id = "stage_test_instance_id"; + [[maybe_unused]] auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer( + (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->set_call_back("decrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* key = try_any_cast(args[0]); + *key = "test"; + auto* ret = try_any_cast(args[1]); + *ret = 0; + }); + sp->enable_processing(); + + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + + RamUserPB ram_user; + ram_user.set_user_id("test_user_id"); + ram_user.set_ak("test_ak"); + ram_user.set_sk("test_sk"); + EncryptionInfoPB encry_info; + encry_info.set_encryption_method("encry_method_test"); + encry_info.set_key_id(1111); + ram_user.mutable_encryption_info()->CopyFrom(encry_info); + + // create instance + { + CreateInstanceRequest req; + req.set_instance_id(instance_id); + req.set_user_id("test_user"); + req.set_name("test_name"); + req.mutable_ram_user()->CopyFrom(ram_user); + req.mutable_obj_info()->CopyFrom(obj); + + CreateInstanceResponse res; + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + // create an internal stage + CreateStageRequest create_stage_request; + StagePB stage; + stage.set_type(StagePB::INTERNAL); + stage.add_mysql_user_name("root"); + stage.add_mysql_user_id("root_id"); + stage.set_stage_id("internal_stage_id"); + create_stage_request.set_cloud_unique_id(cloud_unique_id); + create_stage_request.mutable_stage()->CopyFrom(stage); + CreateStageResponse create_stage_response; + meta_service->create_stage(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &create_stage_request, &create_stage_response, nullptr); + ASSERT_EQ(create_stage_response.status().code(), MetaServiceCode::OK); + + GetStageRequest get_stage_req; + get_stage_req.set_type(StagePB::INTERNAL); + get_stage_req.set_cloud_unique_id(cloud_unique_id); + get_stage_req.set_mysql_user_name("root"); + get_stage_req.set_mysql_user_id("root_id"); + + // get existent internal stage + GetStageResponse res2; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_stage(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &get_stage_req, &res2, nullptr); + ASSERT_EQ(res2.status().code(), MetaServiceCode::OK); + ASSERT_EQ(1, res2.stage().size()); + + LOG(INFO) << "GetStage: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_stage_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_stage_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_stage_del_counter.get_value()); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + // get_iam +TEST(RpcKvBvarTest, DISABLED_GetIam) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + brpc::Controller cntl; + auto cloud_unique_id = "test_cloud_unique_id"; + std::string instance_id = "get_iam_test_instance_id"; + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + std::unique_ptr> defer( + (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->set_call_back("decrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* key = try_any_cast(args[0]); + *key = "test"; + auto* ret = try_any_cast(args[1]); + *ret = 0; + }); + sp->enable_processing(); + + config::arn_id = "iam_arn"; + config::arn_ak = "iam_ak"; + config::arn_sk = "iam_sk"; + + // create instance + { + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + + RamUserPB ram_user; + ram_user.set_user_id("test_user_id"); + ram_user.set_ak("test_ak"); + ram_user.set_sk("test_sk"); + + CreateInstanceRequest req; + req.set_instance_id(instance_id); + req.set_user_id("test_user"); + req.set_name("test_name"); + req.mutable_ram_user()->CopyFrom(ram_user); + req.mutable_obj_info()->CopyFrom(obj); + + CreateInstanceResponse res; + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + GetIamRequest request; + request.set_cloud_unique_id(cloud_unique_id); + GetIamResponse response; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_iam(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &request, + &response, nullptr); + ASSERT_EQ(response.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "GetIam: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_iam_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_iam_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_iam_del_counter.get_value()); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + // alter_iam +TEST(RpcKvBvarTest, DISABLED_AlterIam) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "selectdbselectdbselectdbselectdb"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + + std::string cipher_sk = "JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO"; + std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t"; + + AlterIamRequest req; + req.set_account_id("123"); + req.set_ak("ak1"); + req.set_sk(plain_sk); + + brpc::Controller cntl; + AlterIamResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->alter_iam(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "AlterIam: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_iam_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_iam_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_alter_iam_del_counter.get_value()); +} + // alter_ram_user +TEST(RpcKvBvarTest, DISABLED_AlterRamUser) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + std::string instance_id = "alter_ram_user_instance_id"; + [[maybe_unused]] auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer( + (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->set_call_back("decrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* key = try_any_cast(args[0]); + *key = "test"; + auto* ret = try_any_cast(args[1]); + *ret = 0; + }); + sp->enable_processing(); + + config::arn_id = "iam_arn"; + config::arn_ak = "iam_ak"; + config::arn_sk = "iam_sk"; + + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + + // create instance without ram user + CreateInstanceRequest create_instance_req; + create_instance_req.set_instance_id(instance_id); + create_instance_req.set_user_id("test_user"); + create_instance_req.set_name("test_name"); + create_instance_req.mutable_obj_info()->CopyFrom(obj); + CreateInstanceResponse create_instance_res; + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &create_instance_req, &create_instance_res, nullptr); + ASSERT_EQ(create_instance_res.status().code(), MetaServiceCode::OK); + + // alter ram user + RamUserPB ram_user; + ram_user.set_user_id("test_user_id"); + ram_user.set_ak("test_ak"); + ram_user.set_sk("test_sk"); + AlterRamUserRequest alter_ram_user_request; + alter_ram_user_request.set_instance_id(instance_id); + alter_ram_user_request.mutable_ram_user()->CopyFrom(ram_user); + AlterRamUserResponse alter_ram_user_response; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + meta_service->alter_ram_user(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &alter_ram_user_request, &alter_ram_user_response, nullptr); + + LOG(INFO) << "AlterRamUser: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_ram_user_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_ram_user_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_alter_ram_user_del_counter.get_value()); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + // begin_copy -// finish_copy +TEST(RpcKvBvarTest, DISABLED_BeginCopy) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + auto cloud_unique_id = "test_cloud_unique_id"; + auto stage_id = "test_stage_id"; + int64_t table_id = 100; + std::string instance_id = "copy_job_test_instance_id"; + + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + std::unique_ptr> defer( + (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + + // generate a begin copy request + BeginCopyRequest begin_copy_request; + begin_copy_request.set_cloud_unique_id(cloud_unique_id); + begin_copy_request.set_stage_id(stage_id); + begin_copy_request.set_stage_type(StagePB::EXTERNAL); + begin_copy_request.set_table_id(table_id); + begin_copy_request.set_copy_id("test_copy_id"); + begin_copy_request.set_group_id(0); + begin_copy_request.set_start_time_ms(200); + begin_copy_request.set_timeout_time_ms(300); + for (int i = 0; i < 20; ++i) { + ObjectFilePB object_file_pb; + object_file_pb.set_relative_path("obj_" + std::to_string(i)); + object_file_pb.set_etag("obj_" + std::to_string(i) + "_etag"); + begin_copy_request.add_object_files()->CopyFrom(object_file_pb); + } + BeginCopyResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &begin_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "BeginCopy: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_copy_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_copy_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_begin_copy_del_counter.get_value()); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + // get_copy_job +TEST(RpcKvBvarTest, DISABLED_GetCopyJob) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + const char* cloud_unique_id = "test_cloud_unique_id"; + const char* stage_id = "test_stage_id"; + int64_t table_id = 100; + std::string instance_id = "copy_job_test_instance_id"; + + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + std::unique_ptr> defer( + (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + { + // generate a begin copy request + BeginCopyRequest begin_copy_request; + begin_copy_request.set_cloud_unique_id(cloud_unique_id); + begin_copy_request.set_stage_id(stage_id); + begin_copy_request.set_stage_type(StagePB::EXTERNAL); + begin_copy_request.set_table_id(table_id); + begin_copy_request.set_copy_id("test_copy_id"); + begin_copy_request.set_group_id(0); + begin_copy_request.set_start_time_ms(200); + begin_copy_request.set_timeout_time_ms(300); + for (int i = 0; i < 20; ++i) { + ObjectFilePB object_file_pb; + object_file_pb.set_relative_path("obj_" + std::to_string(i)); + object_file_pb.set_etag("obj_" + std::to_string(i) + "_etag"); + begin_copy_request.add_object_files()->CopyFrom(object_file_pb); + } + BeginCopyResponse res; + meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &begin_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + GetCopyJobRequest get_copy_job_request; + get_copy_job_request.set_cloud_unique_id(cloud_unique_id); + get_copy_job_request.set_stage_id(stage_id); + get_copy_job_request.set_table_id(table_id); + get_copy_job_request.set_copy_id("test_copy_id"); + get_copy_job_request.set_group_id(0); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + GetCopyJobResponse res; + meta_service->get_copy_job(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &get_copy_job_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "GetCopyJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_job_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_copy_job_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_copy_job_del_counter.get_value()); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// finish_copy +TEST(RpcKvBvarTest, DISABLED_FinishCopy) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + const char* cloud_unique_id = "test_cloud_unique_id"; + const char* stage_id = "test_stage_id"; + int64_t table_id = 100; + std::string instance_id = "copy_job_test_instance_id"; + + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + std::unique_ptr> defer( + (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + { + // generate a begin copy request + BeginCopyRequest begin_copy_request; + begin_copy_request.set_cloud_unique_id(cloud_unique_id); + begin_copy_request.set_stage_id(stage_id); + begin_copy_request.set_stage_type(StagePB::EXTERNAL); + begin_copy_request.set_table_id(table_id); + begin_copy_request.set_copy_id("test_copy_id"); + begin_copy_request.set_group_id(0); + begin_copy_request.set_start_time_ms(200); + begin_copy_request.set_timeout_time_ms(300); + for (int i = 0; i < 20; ++i) { + ObjectFilePB object_file_pb; + object_file_pb.set_relative_path("obj_" + std::to_string(i)); + object_file_pb.set_etag("obj_" + std::to_string(i) + "_etag"); + begin_copy_request.add_object_files()->CopyFrom(object_file_pb); + } + BeginCopyResponse res; + meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &begin_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + FinishCopyRequest finish_copy_request; + finish_copy_request.set_cloud_unique_id(cloud_unique_id); + finish_copy_request.set_stage_id(stage_id); + finish_copy_request.set_stage_type(StagePB::EXTERNAL); + finish_copy_request.set_table_id(table_id); + finish_copy_request.set_copy_id("test_copy_id"); + finish_copy_request.set_group_id(0); + finish_copy_request.set_action(FinishCopyRequest::COMMIT); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + FinishCopyResponse res; + meta_service->finish_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &finish_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "FinishCopy: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_finish_copy_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_finish_copy_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_finish_copy_del_counter.get_value()); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + // get_copy_files +TEST(RpcKvBvarTest, DISABLED_GetCopyFiles) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + auto cloud_unique_id = "test_cloud_unique_id"; + auto stage_id = "test_stage_id"; + int64_t table_id = 100; + std::string instance_id = "copy_job_test_instance_id"; + + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + std::unique_ptr> defer( + (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + { + // generate a begin copy request + BeginCopyRequest begin_copy_request; + begin_copy_request.set_cloud_unique_id(cloud_unique_id); + begin_copy_request.set_stage_id(stage_id); + begin_copy_request.set_stage_type(StagePB::EXTERNAL); + begin_copy_request.set_table_id(table_id); + begin_copy_request.set_copy_id("test_copy_id"); + begin_copy_request.set_group_id(0); + begin_copy_request.set_start_time_ms(200); + begin_copy_request.set_timeout_time_ms(300); + for (int i = 0; i < 20; ++i) { + ObjectFilePB object_file_pb; + object_file_pb.set_relative_path("obj_" + std::to_string(i)); + object_file_pb.set_etag("obj_" + std::to_string(i) + "_etag"); + begin_copy_request.add_object_files()->CopyFrom(object_file_pb); + } + BeginCopyResponse res; + meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &begin_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + GetCopyFilesRequest get_copy_file_req; + get_copy_file_req.set_cloud_unique_id(cloud_unique_id); + get_copy_file_req.set_stage_id(stage_id); + get_copy_file_req.set_table_id(table_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + GetCopyFilesResponse res; + meta_service->get_copy_files(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &get_copy_file_req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "GetCopyFiles: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_files_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_copy_files_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_copy_files_del_counter.get_value()); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + // filter_copy_files +TEST(RpcKvBvarTest, DISABLED_FilterCopyFiles) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + auto cloud_unique_id = "test_cloud_unique_id"; + auto stage_id = "test_stage_id"; + int64_t table_id = 100; + std::string instance_id = "copy_job_test_instance_id"; + + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + std::unique_ptr> defer( + (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + { + // generate a begin copy request + BeginCopyRequest begin_copy_request; + begin_copy_request.set_cloud_unique_id(cloud_unique_id); + begin_copy_request.set_stage_id(stage_id); + begin_copy_request.set_stage_type(StagePB::EXTERNAL); + begin_copy_request.set_table_id(table_id); + begin_copy_request.set_copy_id("test_copy_id"); + begin_copy_request.set_group_id(0); + begin_copy_request.set_start_time_ms(200); + begin_copy_request.set_timeout_time_ms(300); + for (int i = 0; i < 20; ++i) { + ObjectFilePB object_file_pb; + object_file_pb.set_relative_path("obj_" + std::to_string(i)); + object_file_pb.set_etag("obj_" + std::to_string(i) + "_etag"); + begin_copy_request.add_object_files()->CopyFrom(object_file_pb); + } + BeginCopyResponse res; + meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &begin_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + FilterCopyFilesRequest request; + request.set_cloud_unique_id(cloud_unique_id); + request.set_stage_id(stage_id); + request.set_table_id(table_id); + for (int i = 0; i < 10; ++i) { + ObjectFilePB object_file; + object_file.set_relative_path("file" + std::to_string(i)); + object_file.set_etag("etag" + std::to_string(i)); + request.add_object_files()->CopyFrom(object_file); + } + FilterCopyFilesResponse res; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->filter_copy_files(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "FilterCopyFiles: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_filter_copy_files_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_filter_copy_files_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_filter_copy_files_del_counter.get_value()); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + // get_cluster_status -// begin_txn -// precommit_txn -// get_rl_task_commit_attach -// reset_rl_progress -// commit_txn -// abort_txn -// get_txn +TEST(RpcKvBvarTest, DISABLED_GetClusterStatus) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + InstanceKeyInfo key_info {mock_instance}; + std::string key; + std::string val; + instance_key(key_info, &key); + + InstanceInfoPB instance; + instance.set_instance_id(mock_instance); + ClusterPB c1; + c1.set_type(ClusterPB::COMPUTE); + c1.set_cluster_name(mock_cluster_name); + c1.set_cluster_id(mock_cluster_id); + c1.add_mysql_user_name()->append("m1"); + c1.set_cluster_status(ClusterStatus::NORMAL); + ClusterPB c2; + c2.set_type(ClusterPB::COMPUTE); + c2.set_cluster_name(mock_cluster_name + "2"); + c2.set_cluster_id(mock_cluster_id + "2"); + c2.add_mysql_user_name()->append("m2"); + c2.set_cluster_status(ClusterStatus::SUSPENDED); + ClusterPB c3; + c3.set_type(ClusterPB::COMPUTE); + c3.set_cluster_name(mock_cluster_name + "3"); + c3.set_cluster_id(mock_cluster_id + "3"); + c3.add_mysql_user_name()->append("m3"); + c3.set_cluster_status(ClusterStatus::TO_RESUME); + instance.add_clusters()->CopyFrom(c1); + instance.add_clusters()->CopyFrom(c2); + instance.add_clusters()->CopyFrom(c3); + val = instance.SerializeAsString(); + + std::unique_ptr txn; + std::string get_val; + TxnErrorCode err = meta_service->txn_kv()->create_txn(&txn); + ASSERT_EQ(err, TxnErrorCode::TXN_OK); + txn->put(key, val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + brpc::Controller cntl; + GetClusterStatusRequest req; + req.add_instance_ids(mock_instance); + GetClusterStatusResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_cluster_status(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + ASSERT_EQ(res.details().at(0).clusters().size(), 3); + + LOG(INFO) << "GetClusterStatus: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_cluster_status_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_cluster_status_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_cluster_status_del_counter.get_value()); +} + // get_current_max_txn_id +TEST(RpcKvBvarTest, DISABLED_GetCurrentMaxTxnId) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + const int64_t db_id = 123; + const std::string label = "test_label123"; + const std::string cloud_unique_id = "test_cloud_unique_id"; + + brpc::Controller begin_txn_cntl; + BeginTxnRequest begin_txn_req; + BeginTxnResponse begin_txn_res; + TxnInfoPB txn_info_pb; + + begin_txn_req.set_cloud_unique_id(cloud_unique_id); + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(12345); + txn_info_pb.set_timeout_ms(36000); + begin_txn_req.mutable_txn_info()->CopyFrom(txn_info_pb); + + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), + &begin_txn_req, &begin_txn_res, nullptr); + ASSERT_EQ(begin_txn_res.status().code(), MetaServiceCode::OK); + + brpc::Controller max_txn_id_cntl; + GetCurrentMaxTxnRequest max_txn_id_req; + GetCurrentMaxTxnResponse max_txn_id_res; + + std::unique_ptr txn; + auto _ = mem_kv->create_txn(&txn); + txn->put("schema change", "val"); + _ = txn->commit(); + + max_txn_id_req.set_cloud_unique_id(cloud_unique_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_current_max_txn_id( + reinterpret_cast<::google::protobuf::RpcController*>(&max_txn_id_cntl), &max_txn_id_req, + &max_txn_id_res, nullptr); + + LOG(INFO) << "GetCurrentMaxTxnId: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_current_max_txn_id_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_current_max_txn_id_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_current_max_txn_id_del_counter.get_value()); +} + // begin_sub_txn +TEST(RpcKvBvarTest, DISABLED_BeginSubTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 98131; + int64_t txn_id = -1; + int64_t t1 = 10; + int64_t t1_index = 100; + int64_t t1_p1 = 11; + int64_t t1_p1_t1 = 12; + int64_t t1_p1_t2 = 13; + int64_t t1_p2 = 14; + int64_t t1_p2_t1 = 15; + int64_t t2 = 16; + std::string label = "test_label"; + std::string label2 = "test_label_0"; + // begin txn + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(t1); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id = res.txn_id(); + } + + // mock rowset and tablet: for sub_txn1 + int64_t sub_txn_id1 = txn_id; + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p1, t1_p1_t1, sub_txn_id1); + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p1, t1_p1_t2, sub_txn_id1); + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p2, t1_p2_t1, sub_txn_id1); + + brpc::Controller cntl; + BeginSubTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_sub_txn_num(0); + req.set_db_id(db_id); + req.set_label(label2); + req.mutable_table_ids()->Add(t1); + req.mutable_table_ids()->Add(t2); + BeginSubTxnResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->begin_sub_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "BeginSubTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_sub_txn_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_sub_txn_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_begin_sub_txn_del_counter.get_value()); +} + // abort_sub_txn +TEST(RpcKvBvarTest, DISABLED_AbortSubTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 98131; + int64_t txn_id = -1; + int64_t t1 = 10; + int64_t t1_index = 100; + int64_t t1_p1 = 11; + int64_t t1_p1_t1 = 12; + int64_t t1_p1_t2 = 13; + int64_t t1_p2 = 14; + int64_t t1_p2_t1 = 15; + int64_t t2 = 16; + std::string label = "test_label"; + std::string label2 = "test_label_0"; + // begin txn + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(t1); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id = res.txn_id(); + } + + // mock rowset and tablet: for sub_txn1 + int64_t sub_txn_id1 = txn_id; + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p1, t1_p1_t1, sub_txn_id1); + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p1, t1_p1_t2, sub_txn_id1); + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p2, t1_p2_t1, sub_txn_id1); + brpc::Controller cntl; + { + BeginSubTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_sub_txn_num(0); + req.set_db_id(db_id); + req.set_label(label2); + req.mutable_table_ids()->Add(t1); + req.mutable_table_ids()->Add(t2); + BeginSubTxnResponse res; + + meta_service->begin_sub_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + } + + AbortSubTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_sub_txn_num(2); + req.set_sub_txn_id(sub_txn_id1); + req.set_db_id(db_id); + req.mutable_table_ids()->Add(t1); + req.mutable_table_ids()->Add(t2); + AbortSubTxnResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->abort_sub_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "AbortSubTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_sub_txn_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_sub_txn_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_abort_sub_txn_del_counter.get_value()); +} + // abort_txn_with_coordinator +TEST(RpcKvBvarTest, DISABLED_AbortTxnWithCoordinator) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + const int64_t db_id = 666; + const int64_t table_id = 777; + const std::string label = "test_label"; + const std::string cloud_unique_id = "test_cloud_unique_id"; + const int64_t coordinator_id = 15623; + int64_t cur_time = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + std::string host = "127.0.0.1:15586"; + int64_t txn_id = -1; + + brpc::Controller begin_txn_cntl; + BeginTxnRequest begin_txn_req; + BeginTxnResponse begin_txn_res; + TxnInfoPB txn_info_pb; + TxnCoordinatorPB coordinator; + + begin_txn_req.set_cloud_unique_id(cloud_unique_id); + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.set_timeout_ms(36000); + coordinator.set_id(coordinator_id); + coordinator.set_ip(host); + coordinator.set_sourcetype(::doris::cloud::TxnSourceTypePB::TXN_SOURCE_TYPE_BE); + coordinator.set_start_time(cur_time); + txn_info_pb.mutable_coordinator()->CopyFrom(coordinator); + begin_txn_req.mutable_txn_info()->CopyFrom(txn_info_pb); + + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), + &begin_txn_req, &begin_txn_res, nullptr); + ASSERT_EQ(begin_txn_res.status().code(), MetaServiceCode::OK); + txn_id = begin_txn_res.txn_id(); + ASSERT_GT(txn_id, -1); + + brpc::Controller abort_txn_cntl; + AbortTxnWithCoordinatorRequest abort_txn_req; + AbortTxnWithCoordinatorResponse abort_txn_resp; + + abort_txn_req.set_id(coordinator_id); + abort_txn_req.set_ip(host); + abort_txn_req.set_start_time(cur_time + 3600); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->abort_txn_with_coordinator( + reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), &abort_txn_req, + &abort_txn_resp, nullptr); + + LOG(INFO) << "AbortTxnWithCoordinator: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ + << ", " << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_txn_with_coordinator_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_abort_txn_with_coordinator_del_counter.get_value()); +} + // check_txn_conflict +TEST(RpcKvBvarTest, DISABLED_CheckTxnConflict) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + const int64_t db_id = 666; + const int64_t table_id = 777; + const std::string label = "test_label"; + const std::string cloud_unique_id = "test_cloud_unique_id"; + const int64_t coordinator_id = 15623; + int64_t cur_time = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + std::string host = "127.0.0.1:15586"; + int64_t txn_id = -1; + + brpc::Controller begin_txn_cntl; + BeginTxnRequest begin_txn_req; + BeginTxnResponse begin_txn_res; + TxnInfoPB txn_info_pb; + TxnCoordinatorPB coordinator; + + begin_txn_req.set_cloud_unique_id(cloud_unique_id); + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.set_timeout_ms(36000); + coordinator.set_id(coordinator_id); + coordinator.set_ip(host); + coordinator.set_sourcetype(::doris::cloud::TxnSourceTypePB::TXN_SOURCE_TYPE_BE); + coordinator.set_start_time(cur_time); + txn_info_pb.mutable_coordinator()->CopyFrom(coordinator); + begin_txn_req.mutable_txn_info()->CopyFrom(txn_info_pb); + + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), + &begin_txn_req, &begin_txn_res, nullptr); + + brpc::Controller abort_txn_conflict_cntl; + CheckTxnConflictRequest check_txn_conflict_req; + CheckTxnConflictResponse check_txn_conflict_res; + + check_txn_conflict_req.set_cloud_unique_id(cloud_unique_id); + check_txn_conflict_req.set_db_id(db_id); + check_txn_conflict_req.set_end_txn_id(txn_id + 1); + check_txn_conflict_req.add_table_ids(table_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->check_txn_conflict( + reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), + &check_txn_conflict_req, &check_txn_conflict_res, nullptr); + + LOG(INFO) << "CheckTxnConflict: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_check_txn_conflict_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_check_txn_conflict_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_check_txn_conflict_del_counter.get_value()); +} + // clean_txn_label -// get_txn_id +TEST(RpcKvBvarTest, DISABLED_CleanTxnLabel) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 1987211; + const std::string& label = "test_clean_label"; + brpc::Controller cntl; + { + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(1234); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + } + CleanTxnLabelRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_db_id(db_id); + req.add_labels(label); + CleanTxnLabelResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->clean_txn_label(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "CleanTxnLabel: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_clean_txn_label_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_clean_txn_label_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_clean_txn_label_del_counter.get_value()); +} + +// get_rl_task_commit_attach +TEST(RpcKvBvarTest, GetRlTaskCommitAttach) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + LOG(INFO) << "GetRlTaskCommitAttach: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ + << ", " << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_rl_task_commit_attach_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_rl_task_commit_attach_del_counter.get_value()); +} + +// reset_rl_progress +TEST(RpcKvBvarTest, ResetRlProgress) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + LOG(INFO) << "ResetRlProgress: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_reset_rl_progress_get_counter.get_value()); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_reset_rl_progress_put_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_reset_rl_progress_del_counter.get_value()); +} } // namespace doris::cloud From 62de960ef4e48ba4cafb0b92ed93c22a84276965 Mon Sep 17 00:00:00 2001 From: koarz Date: Wed, 18 Jun 2025 17:05:36 +0800 Subject: [PATCH 03/23] remove useless code --- cloud/src/common/bvars.cpp | 67 ------ cloud/src/common/bvars.h | 67 ------ cloud/src/common/stats.h | 10 +- cloud/src/meta-service/meta_service.cpp | 116 +++++----- cloud/src/meta-service/meta_service_helper.h | 83 ++++--- cloud/src/meta-service/meta_service_job.cpp | 4 +- .../meta-service/meta_service_partition.cpp | 18 +- .../meta-service/meta_service_resource.cpp | 63 +++--- cloud/src/meta-service/meta_service_txn.cpp | 77 ++++--- cloud/test/rpc_kv_bvar_test.cpp | 208 +++++------------- 10 files changed, 235 insertions(+), 478 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 4b691649eea9cb..dcd082ffc3b3cc 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -191,36 +191,24 @@ BvarStatusWithTag g_bvar_max_rowsets_with_useless_delete_bitmap_version // rpc kv rw count // get_rowset bvar::Adder g_bvar_rpc_kv_get_rowset_get_counter("rpc_kv","get_rowset_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_rowset_put_counter("rpc_kv","get_rowset_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_rowset_del_counter("rpc_kv","get_rowset_del_counter"); // get_version bvar::Adder g_bvar_rpc_kv_get_version_get_counter("rpc_kv","get_version_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_version_put_counter("rpc_kv","get_version_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_version_del_counter("rpc_kv","get_version_del_counter"); // get_schema_dict bvar::Adder g_bvar_rpc_kv_get_schema_dict_get_counter("rpc_kv","get_schema_dict_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_schema_dict_put_counter("rpc_kv","get_schema_dict_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_schema_dict_del_counter("rpc_kv","get_schema_dict_del_counter"); // create_tablets bvar::Adder g_bvar_rpc_kv_create_tablets_get_counter("rpc_kv","create_tablets_get_counter"); bvar::Adder g_bvar_rpc_kv_create_tablets_put_counter("rpc_kv","create_tablets_put_counter"); -bvar::Adder g_bvar_rpc_kv_create_tablets_del_counter("rpc_kv","create_tablets_del_counter"); // update_tablet bvar::Adder g_bvar_rpc_kv_update_tablet_get_counter("rpc_kv","update_tablet_get_counter"); bvar::Adder g_bvar_rpc_kv_update_tablet_put_counter("rpc_kv","update_tablet_put_counter"); -bvar::Adder g_bvar_rpc_kv_update_tablet_del_counter("rpc_kv","update_tablet_del_counter"); // update_tablet_schema bvar::Adder g_bvar_rpc_kv_update_tablet_schema_get_counter("rpc_kv","update_tablet_schema_get_counter"); bvar::Adder g_bvar_rpc_kv_update_tablet_schema_put_counter("rpc_kv","update_tablet_schema_put_counter"); -bvar::Adder g_bvar_rpc_kv_update_tablet_schema_del_counter("rpc_kv","update_tablet_schema_del_counter"); // get_tablet bvar::Adder g_bvar_rpc_kv_get_tablet_get_counter("rpc_kv","get_tablet_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_tablet_put_counter("rpc_kv","get_tablet_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_tablet_del_counter("rpc_kv","get_tablet_del_counter"); // prepare_rowset bvar::Adder g_bvar_rpc_kv_prepare_rowset_get_counter("rpc_kv","prepare_rowset_get_counter"); bvar::Adder g_bvar_rpc_kv_prepare_rowset_put_counter("rpc_kv","prepare_rowset_put_counter"); -bvar::Adder g_bvar_rpc_kv_prepare_rowset_del_counter("rpc_kv","prepare_rowset_del_counter"); // commit_rowset bvar::Adder g_bvar_rpc_kv_commit_rowset_get_counter("rpc_kv","commit_rowset_get_counter"); bvar::Adder g_bvar_rpc_kv_commit_rowset_put_counter("rpc_kv","commit_rowset_put_counter"); @@ -228,19 +216,14 @@ bvar::Adder g_bvar_rpc_kv_commit_rowset_del_counter("rpc_kv","commit_ro // update_tmp_rowset bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_get_counter("rpc_kv","update_tmp_rowset_get_counter"); bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_put_counter("rpc_kv","update_tmp_rowset_put_counter"); -bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_del_counter("rpc_kv","update_tmp_rowset_del_counter"); // get_tablet_stats bvar::Adder g_bvar_rpc_kv_get_tablet_stats_get_counter("rpc_kv","get_tablet_stats_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_tablet_stats_put_counter("rpc_kv","get_tablet_stats_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_tablet_stats_del_counter("rpc_kv","get_tablet_stats_del_counter"); // update_delete_bitmap bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_get_counter("rpc_kv","update_delete_bitmap_get_counter"); bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_put_counter("rpc_kv","update_delete_bitmap_put_counter"); bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_del_counter("rpc_kv","update_delete_bitmap_del_counter"); // get_delete_bitmap bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_get_counter("rpc_kv","get_delete_bitmap_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_put_counter("rpc_kv","get_delete_bitmap_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_del_counter("rpc_kv","get_delete_bitmap_del_counter"); // get_delete_bitmap_update_lock bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter("rpc_kv","get_delete_bitmap_update_lock_get_counter"); bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter("rpc_kv","get_delete_bitmap_update_lock_put_counter"); @@ -250,13 +233,10 @@ bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter( bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter("rpc_kv","remove_delete_bitmap_update_lock_put_counter"); bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter("rpc_kv","remove_delete_bitmap_update_lock_del_counter"); // remove_delete_bitmap -bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_get_counter("rpc_kv","remove_delete_bitmap_get_counter"); -bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_put_counter("rpc_kv","remove_delete_bitmap_put_counter"); bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_del_counter("rpc_kv","remove_delete_bitmap_del_counter"); // start_tablet_job bvar::Adder g_bvar_rpc_kv_start_tablet_job_get_counter("rpc_kv","start_tablet_job_get_counter"); bvar::Adder g_bvar_rpc_kv_start_tablet_job_put_counter("rpc_kv","start_tablet_job_put_counter"); -bvar::Adder g_bvar_rpc_kv_start_tablet_job_del_counter("rpc_kv","start_tablet_job_del_counter"); // finish_tablet_job bvar::Adder g_bvar_rpc_kv_finish_tablet_job_get_counter("rpc_kv","finish_tablet_job_get_counter"); bvar::Adder g_bvar_rpc_kv_finish_tablet_job_put_counter("rpc_kv","finish_tablet_job_put_counter"); @@ -264,7 +244,6 @@ bvar::Adder g_bvar_rpc_kv_finish_tablet_job_del_counter("rpc_kv","finis // prepare_index bvar::Adder g_bvar_rpc_kv_prepare_index_get_counter("rpc_kv","prepare_index_get_counter"); bvar::Adder g_bvar_rpc_kv_prepare_index_put_counter("rpc_kv","prepare_index_put_counter"); -bvar::Adder g_bvar_rpc_kv_prepare_index_del_counter("rpc_kv","prepare_index_del_counter"); // commit_index bvar::Adder g_bvar_rpc_kv_commit_index_get_counter("rpc_kv","commit_index_get_counter"); bvar::Adder g_bvar_rpc_kv_commit_index_put_counter("rpc_kv","commit_index_put_counter"); @@ -272,11 +251,9 @@ bvar::Adder g_bvar_rpc_kv_commit_index_del_counter("rpc_kv","commit_ind // drop_index bvar::Adder g_bvar_rpc_kv_drop_index_get_counter("rpc_kv","drop_index_get_counter"); bvar::Adder g_bvar_rpc_kv_drop_index_put_counter("rpc_kv","drop_index_put_counter"); -bvar::Adder g_bvar_rpc_kv_drop_index_del_counter("rpc_kv","drop_index_del_counter"); // prepare_partition bvar::Adder g_bvar_rpc_kv_prepare_partition_get_counter("rpc_kv","prepare_partition_get_counter"); bvar::Adder g_bvar_rpc_kv_prepare_partition_put_counter("rpc_kv","prepare_partition_put_counter"); -bvar::Adder g_bvar_rpc_kv_prepare_partition_del_counter("rpc_kv","prepare_partition_del_counter"); // commit_partition bvar::Adder g_bvar_rpc_kv_commit_partition_get_counter("rpc_kv","commit_partition_get_counter"); bvar::Adder g_bvar_rpc_kv_commit_partition_put_counter("rpc_kv","commit_partition_put_counter"); @@ -284,15 +261,10 @@ bvar::Adder g_bvar_rpc_kv_commit_partition_del_counter("rpc_kv","commit // drop_partition bvar::Adder g_bvar_rpc_kv_drop_partition_get_counter("rpc_kv","drop_partition_get_counter"); bvar::Adder g_bvar_rpc_kv_drop_partition_put_counter("rpc_kv","drop_partition_put_counter"); -bvar::Adder g_bvar_rpc_kv_drop_partition_del_counter("rpc_kv","drop_partition_del_counter"); // check_kv bvar::Adder g_bvar_rpc_kv_check_kv_get_counter("rpc_kv","check_kv_get_counter"); -bvar::Adder g_bvar_rpc_kv_check_kv_put_counter("rpc_kv","check_kv_put_counter"); -bvar::Adder g_bvar_rpc_kv_check_kv_del_counter("rpc_kv","check_kv_del_counter"); // get_obj_store_info bvar::Adder g_bvar_rpc_kv_get_obj_store_info_get_counter("rpc_kv","get_obj_store_info_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_obj_store_info_put_counter("rpc_kv","get_obj_store_info_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_obj_store_info_del_counter("rpc_kv","get_obj_store_info_del_counter"); // alter_storage_vault bvar::Adder g_bvar_rpc_kv_alter_storage_vault_get_counter("rpc_kv","alter_storage_vault_get_counter"); bvar::Adder g_bvar_rpc_kv_alter_storage_vault_put_counter("rpc_kv","alter_storage_vault_put_counter"); @@ -300,83 +272,55 @@ bvar::Adder g_bvar_rpc_kv_alter_storage_vault_del_counter("rpc_kv","alt // alter_obj_store_info bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_get_counter("rpc_kv","alter_obj_store_info_get_counter"); bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_put_counter("rpc_kv","alter_obj_store_info_put_counter"); -bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_del_counter("rpc_kv","alter_obj_store_info_del_counter"); // update_ak_sk bvar::Adder g_bvar_rpc_kv_update_ak_sk_get_counter("rpc_kv","update_ak_sk_get_counter"); bvar::Adder g_bvar_rpc_kv_update_ak_sk_put_counter("rpc_kv","update_ak_sk_put_counter"); -bvar::Adder g_bvar_rpc_kv_update_ak_sk_del_counter("rpc_kv","update_ak_sk_del_counter"); // create_instance bvar::Adder g_bvar_rpc_kv_create_instance_get_counter("rpc_kv","create_instance_get_counter"); bvar::Adder g_bvar_rpc_kv_create_instance_put_counter("rpc_kv","create_instance_put_counter"); -bvar::Adder g_bvar_rpc_kv_create_instance_del_counter("rpc_kv","create_instance_del_counter"); // get_instance bvar::Adder g_bvar_rpc_kv_get_instance_get_counter("rpc_kv","get_instance_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_instance_put_counter("rpc_kv","get_instance_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_instance_del_counter("rpc_kv","get_instance_del_counter"); // alter_cluster bvar::Adder g_bvar_rpc_kv_alter_cluster_get_counter("rpc_kv","alter_cluster_get_counter"); -bvar::Adder g_bvar_rpc_kv_alter_cluster_put_counter("rpc_kv","alter_cluster_put_counter"); -bvar::Adder g_bvar_rpc_kv_alter_cluster_del_counter("rpc_kv","alter_cluster_del_counter"); // get_cluster bvar::Adder g_bvar_rpc_kv_get_cluster_get_counter("rpc_kv","get_cluster_get_counter"); bvar::Adder g_bvar_rpc_kv_get_cluster_put_counter("rpc_kv","get_cluster_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_cluster_del_counter("rpc_kv","get_cluster_del_counter"); // create_stage bvar::Adder g_bvar_rpc_kv_create_stage_get_counter("rpc_kv","create_stage_get_counter"); bvar::Adder g_bvar_rpc_kv_create_stage_put_counter("rpc_kv","create_stage_put_counter"); -bvar::Adder g_bvar_rpc_kv_create_stage_del_counter("rpc_kv","create_stage_del_counter"); // get_stage bvar::Adder g_bvar_rpc_kv_get_stage_get_counter("rpc_kv","get_stage_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_stage_put_counter("rpc_kv","get_stage_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_stage_del_counter("rpc_kv","get_stage_del_counter"); // get_iam bvar::Adder g_bvar_rpc_kv_get_iam_get_counter("rpc_kv","get_iam_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_iam_put_counter("rpc_kv","get_iam_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_iam_del_counter("rpc_kv","get_iam_del_counter"); // alter_iam bvar::Adder g_bvar_rpc_kv_alter_iam_get_counter("rpc_kv","alter_iam_get_counter"); bvar::Adder g_bvar_rpc_kv_alter_iam_put_counter("rpc_kv","alter_iam_put_counter"); -bvar::Adder g_bvar_rpc_kv_alter_iam_del_counter("rpc_kv","alter_iam_del_counter"); // alter_ram_user bvar::Adder g_bvar_rpc_kv_alter_ram_user_get_counter("rpc_kv","alter_ram_user_get_counter"); bvar::Adder g_bvar_rpc_kv_alter_ram_user_put_counter("rpc_kv","alter_ram_user_put_counter"); -bvar::Adder g_bvar_rpc_kv_alter_ram_user_del_counter("rpc_kv","alter_ram_user_del_counter"); // begin_copy bvar::Adder g_bvar_rpc_kv_begin_copy_get_counter("rpc_kv","begin_copy_get_counter"); bvar::Adder g_bvar_rpc_kv_begin_copy_put_counter("rpc_kv","begin_copy_put_counter"); -bvar::Adder g_bvar_rpc_kv_begin_copy_del_counter("rpc_kv","begin_copy_del_counter"); // finish_copy bvar::Adder g_bvar_rpc_kv_finish_copy_get_counter("rpc_kv","finish_copy_get_counter"); bvar::Adder g_bvar_rpc_kv_finish_copy_put_counter("rpc_kv","finish_copy_put_counter"); bvar::Adder g_bvar_rpc_kv_finish_copy_del_counter("rpc_kv","finish_copy_del_counter"); // get_copy_job bvar::Adder g_bvar_rpc_kv_get_copy_job_get_counter("rpc_kv","get_copy_job_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_copy_job_put_counter("rpc_kv","get_copy_job_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_copy_job_del_counter("rpc_kv","get_copy_job_del_counter"); // get_copy_files bvar::Adder g_bvar_rpc_kv_get_copy_files_get_counter("rpc_kv","get_copy_files_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_copy_files_put_counter("rpc_kv","get_copy_files_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_copy_files_del_counter("rpc_kv","get_copy_files_del_counter"); // filter_copy_files bvar::Adder g_bvar_rpc_kv_filter_copy_files_get_counter("rpc_kv","filter_copy_files_get_counter"); -bvar::Adder g_bvar_rpc_kv_filter_copy_files_put_counter("rpc_kv","filter_copy_files_put_counter"); -bvar::Adder g_bvar_rpc_kv_filter_copy_files_del_counter("rpc_kv","filter_copy_files_del_counter"); // get_cluster_status bvar::Adder g_bvar_rpc_kv_get_cluster_status_get_counter("rpc_kv","get_cluster_status_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_cluster_status_put_counter("rpc_kv","get_cluster_status_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_cluster_status_del_counter("rpc_kv","get_cluster_status_del_counter"); // begin_txn bvar::Adder g_bvar_rpc_kv_begin_txn_get_counter("rpc_kv","begin_txn_get_counter"); bvar::Adder g_bvar_rpc_kv_begin_txn_put_counter("rpc_kv","begin_txn_put_counter"); -bvar::Adder g_bvar_rpc_kv_begin_txn_del_counter("rpc_kv","begin_txn_del_counter"); // precommit_txn bvar::Adder g_bvar_rpc_kv_precommit_txn_get_counter("rpc_kv","precommit_txn_get_counter"); bvar::Adder g_bvar_rpc_kv_precommit_txn_put_counter("rpc_kv","precommit_txn_put_counter"); -bvar::Adder g_bvar_rpc_kv_precommit_txn_del_counter("rpc_kv","precommit_txn_del_counter"); // get_rl_task_commit_attach bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter("rpc_kv","get_rl_task_commit_attach_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_put_counter("rpc_kv","get_rl_task_commit_attach_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_del_counter("rpc_kv","get_rl_task_commit_attach_del_counter"); // reset_rl_progress bvar::Adder g_bvar_rpc_kv_reset_rl_progress_get_counter("rpc_kv","reset_rl_progress_get_counter"); bvar::Adder g_bvar_rpc_kv_reset_rl_progress_put_counter("rpc_kv","reset_rl_progress_put_counter"); @@ -391,12 +335,8 @@ bvar::Adder g_bvar_rpc_kv_abort_txn_put_counter("rpc_kv","abort_txn_put bvar::Adder g_bvar_rpc_kv_abort_txn_del_counter("rpc_kv","abort_txn_del_counter"); // get_txn bvar::Adder g_bvar_rpc_kv_get_txn_get_counter("rpc_kv","get_txn_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_txn_put_counter("rpc_kv","get_txn_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_txn_del_counter("rpc_kv","get_txn_del_counter"); // get_current_max_txn_id bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_get_counter("rpc_kv","get_current_max_txn_id_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_put_counter("rpc_kv","get_current_max_txn_id_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_del_counter("rpc_kv","get_current_max_txn_id_del_counter"); // begin_sub_txn bvar::Adder g_bvar_rpc_kv_begin_sub_txn_get_counter("rpc_kv","begin_sub_txn_get_counter"); bvar::Adder g_bvar_rpc_kv_begin_sub_txn_put_counter("rpc_kv","begin_sub_txn_put_counter"); @@ -404,22 +344,15 @@ bvar::Adder g_bvar_rpc_kv_begin_sub_txn_del_counter("rpc_kv","begin_sub // abort_sub_txn bvar::Adder g_bvar_rpc_kv_abort_sub_txn_get_counter("rpc_kv","abort_sub_txn_get_counter"); bvar::Adder g_bvar_rpc_kv_abort_sub_txn_put_counter("rpc_kv","abort_sub_txn_put_counter"); -bvar::Adder g_bvar_rpc_kv_abort_sub_txn_del_counter("rpc_kv","abort_sub_txn_del_counter"); // abort_txn_with_coordinator bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter("rpc_kv","abort_txn_with_coordinator_get_counter"); -bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_put_counter("rpc_kv","abort_txn_with_coordinator_put_counter"); -bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_del_counter("rpc_kv","abort_txn_with_coordinator_del_counter"); // check_txn_conflict bvar::Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv","check_txn_conflict_get_counter"); -bvar::Adder g_bvar_rpc_kv_check_txn_conflict_put_counter("rpc_kv","check_txn_conflict_put_counter"); -bvar::Adder g_bvar_rpc_kv_check_txn_conflict_del_counter("rpc_kv","check_txn_conflict_del_counter"); // clean_txn_label bvar::Adder g_bvar_rpc_kv_clean_txn_label_get_counter("rpc_kv","clean_txn_label_get_counter"); bvar::Adder g_bvar_rpc_kv_clean_txn_label_put_counter("rpc_kv","clean_txn_label_put_counter"); bvar::Adder g_bvar_rpc_kv_clean_txn_label_del_counter("rpc_kv","clean_txn_label_del_counter"); // get_txn_id bvar::Adder g_bvar_rpc_kv_get_txn_id_get_counter("rpc_kv","get_txn_id_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_txn_id_put_counter("rpc_kv","get_txn_id_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_txn_id_del_counter("rpc_kv","get_txn_id_del_counter"); // clang-format on diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index af2e8455770993..25f76e38ae1844 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -257,146 +257,90 @@ extern BvarStatusWithTag g_bvar_max_rowsets_with_useless_delete_bitmap_ // rpc kv extern bvar::Adder g_bvar_rpc_kv_get_rowset_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_rowset_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_rowset_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_version_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_version_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_version_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_del_counter; extern bvar::Adder g_bvar_rpc_kv_create_tablets_get_counter; extern bvar::Adder g_bvar_rpc_kv_create_tablets_put_counter; -extern bvar::Adder g_bvar_rpc_kv_create_tablets_del_counter; extern bvar::Adder g_bvar_rpc_kv_update_tablet_get_counter; extern bvar::Adder g_bvar_rpc_kv_update_tablet_put_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tablet_del_counter; extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_get_counter; extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_put_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_tablet_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_tablet_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_tablet_del_counter; extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_get_counter; extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_put_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_del_counter; extern bvar::Adder g_bvar_rpc_kv_commit_rowset_get_counter; extern bvar::Adder g_bvar_rpc_kv_commit_rowset_put_counter; extern bvar::Adder g_bvar_rpc_kv_commit_rowset_del_counter; extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_get_counter; extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_put_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_del_counter; extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_get_counter; extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_put_counter; extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter; extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter; extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter; extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter; extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter; extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter; -extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_get_counter; -extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_put_counter; extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_del_counter; extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_get_counter; extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_put_counter; -extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_del_counter; extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_get_counter; extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_put_counter; extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_del_counter; extern bvar::Adder g_bvar_rpc_kv_prepare_index_get_counter; extern bvar::Adder g_bvar_rpc_kv_prepare_index_put_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_index_del_counter; extern bvar::Adder g_bvar_rpc_kv_commit_index_get_counter; extern bvar::Adder g_bvar_rpc_kv_commit_index_put_counter; extern bvar::Adder g_bvar_rpc_kv_commit_index_del_counter; extern bvar::Adder g_bvar_rpc_kv_drop_index_get_counter; extern bvar::Adder g_bvar_rpc_kv_drop_index_put_counter; -extern bvar::Adder g_bvar_rpc_kv_drop_index_del_counter; extern bvar::Adder g_bvar_rpc_kv_prepare_partition_get_counter; extern bvar::Adder g_bvar_rpc_kv_prepare_partition_put_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_partition_del_counter; extern bvar::Adder g_bvar_rpc_kv_commit_partition_get_counter; extern bvar::Adder g_bvar_rpc_kv_commit_partition_put_counter; extern bvar::Adder g_bvar_rpc_kv_commit_partition_del_counter; extern bvar::Adder g_bvar_rpc_kv_drop_partition_get_counter; extern bvar::Adder g_bvar_rpc_kv_drop_partition_put_counter; -extern bvar::Adder g_bvar_rpc_kv_drop_partition_del_counter; extern bvar::Adder g_bvar_rpc_kv_check_kv_get_counter; -extern bvar::Adder g_bvar_rpc_kv_check_kv_put_counter; -extern bvar::Adder g_bvar_rpc_kv_check_kv_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_del_counter; extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_get_counter; extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_put_counter; extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_del_counter; extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_get_counter; extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_put_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_del_counter; extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_get_counter; extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_put_counter; -extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_del_counter; extern bvar::Adder g_bvar_rpc_kv_create_instance_get_counter; extern bvar::Adder g_bvar_rpc_kv_create_instance_put_counter; -extern bvar::Adder g_bvar_rpc_kv_create_instance_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_instance_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_instance_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_instance_del_counter; extern bvar::Adder g_bvar_rpc_kv_alter_cluster_get_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_cluster_put_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_cluster_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_cluster_get_counter; extern bvar::Adder g_bvar_rpc_kv_get_cluster_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_cluster_del_counter; extern bvar::Adder g_bvar_rpc_kv_create_stage_get_counter; extern bvar::Adder g_bvar_rpc_kv_create_stage_put_counter; -extern bvar::Adder g_bvar_rpc_kv_create_stage_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_stage_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_stage_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_stage_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_iam_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_iam_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_iam_del_counter; extern bvar::Adder g_bvar_rpc_kv_alter_iam_get_counter; extern bvar::Adder g_bvar_rpc_kv_alter_iam_put_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_iam_del_counter; extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_get_counter; extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_put_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_del_counter; extern bvar::Adder g_bvar_rpc_kv_begin_copy_get_counter; extern bvar::Adder g_bvar_rpc_kv_begin_copy_put_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_copy_del_counter; extern bvar::Adder g_bvar_rpc_kv_finish_copy_get_counter; extern bvar::Adder g_bvar_rpc_kv_finish_copy_put_counter; extern bvar::Adder g_bvar_rpc_kv_finish_copy_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_copy_job_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_copy_job_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_copy_job_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_copy_files_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_copy_files_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_copy_files_del_counter; extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_get_counter; -extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_put_counter; -extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_del_counter; extern bvar::Adder g_bvar_rpc_kv_begin_txn_get_counter; extern bvar::Adder g_bvar_rpc_kv_begin_txn_put_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_txn_del_counter; extern bvar::Adder g_bvar_rpc_kv_precommit_txn_get_counter; extern bvar::Adder g_bvar_rpc_kv_precommit_txn_put_counter; -extern bvar::Adder g_bvar_rpc_kv_precommit_txn_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_del_counter; extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_get_counter; extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_put_counter; extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_del_counter; @@ -407,26 +351,15 @@ extern bvar::Adder g_bvar_rpc_kv_abort_txn_get_counter; extern bvar::Adder g_bvar_rpc_kv_abort_txn_put_counter; extern bvar::Adder g_bvar_rpc_kv_abort_txn_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_txn_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_txn_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_txn_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_del_counter; extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_get_counter; extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_put_counter; extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_del_counter; extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_get_counter; extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_put_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_del_counter; extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_put_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_del_counter; extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_get_counter; -extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_put_counter; -extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_del_counter; extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_get_counter; extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_put_counter; extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_del_counter; extern bvar::Adder g_bvar_rpc_kv_get_txn_id_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_txn_id_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_txn_id_del_counter; diff --git a/cloud/src/common/stats.h b/cloud/src/common/stats.h index feda1552969d7b..3e9d9a5df1100f 100644 --- a/cloud/src/common/stats.h +++ b/cloud/src/common/stats.h @@ -23,13 +23,9 @@ namespace doris::cloud { struct KVStats { - KVStats(bvar::Adder& get_counter, bvar::Adder& put_counter, - bvar::Adder& del_counter) - : get_counter(get_counter), put_counter(put_counter), del_counter(del_counter) {} - - bvar::Adder& get_counter; - bvar::Adder& put_counter; - bvar::Adder& del_counter; + int64_t get_counter {}; + int64_t put_counter {}; + int64_t del_counter {}; }; } // namespace doris::cloud \ No newline at end of file diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index cf5503fa1262dc..9f556de84455f0 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -215,7 +215,7 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, return; } - RPC_PREPROCESS(get_version); + RPC_PREPROCESS(get_version, get); std::string cloud_unique_id; if (request->has_cloud_unique_id()) { cloud_unique_id = request->cloud_unique_id(); @@ -309,7 +309,7 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr const GetVersionRequest* request, GetVersionResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_version); + RPC_PREPROCESS(get_version, get); std::string cloud_unique_id; if (request->has_cloud_unique_id()) { @@ -365,11 +365,8 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr code = cast_as(err); break; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); - }); + std::unique_ptr> defer_stats( + (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); for (size_t i = response->versions_size(); i < num_acquired; i += BATCH_SIZE) { size_t limit = (i + BATCH_SIZE < num_acquired) ? i + BATCH_SIZE : num_acquired; version_keys.clear(); @@ -468,9 +465,8 @@ void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); }); std::string rs_key, rs_val; @@ -585,7 +581,7 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll const CreateTabletsRequest* request, CreateTabletsResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(create_tablets); + RPC_PREPROCESS(create_tablets, get, put); if (request->tablet_metas_size() == 0) { msg = "no tablet meta"; @@ -609,11 +605,8 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll msg = fmt::format("failed to create txn"); return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); - }); + std::unique_ptr> defer_stats( + (int*)0x01, [&](int*) { stats.get_counter += txn0->num_get_keys(); }); InstanceKeyInfo key_info {instance_id}; std::string key; @@ -746,7 +739,7 @@ void MetaServiceImpl::update_tablet(::google::protobuf::RpcController* controlle const UpdateTabletRequest* request, UpdateTabletResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(update_tablet); + RPC_PREPROCESS(update_tablet, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -851,7 +844,7 @@ void MetaServiceImpl::update_tablet_schema(::google::protobuf::RpcController* co UpdateTabletSchemaResponse* response, ::google::protobuf::Closure* done) { DCHECK(false) << "should not call update_tablet_schema"; - RPC_PREPROCESS(update_tablet_schema); + RPC_PREPROCESS(update_tablet_schema, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -923,7 +916,7 @@ void MetaServiceImpl::update_tablet_schema(::google::protobuf::RpcController* co void MetaServiceImpl::get_tablet(::google::protobuf::RpcController* controller, const GetTabletRequest* request, GetTabletResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_tablet); + RPC_PREPROCESS(get_tablet, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -1142,7 +1135,7 @@ void MetaServiceImpl::prepare_rowset(::google::protobuf::RpcController* controll const CreateRowsetRequest* request, CreateRowsetResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(prepare_rowset); + RPC_PREPROCESS(prepare_rowset, get, put); if (!request->has_rowset_meta()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "no rowset meta"; @@ -1287,7 +1280,7 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle const CreateRowsetRequest* request, CreateRowsetResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(commit_rowset); + RPC_PREPROCESS(commit_rowset, get, put, del); if (!request->has_rowset_meta()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "no rowset meta"; @@ -1447,7 +1440,7 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr const CreateRowsetRequest* request, CreateRowsetResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(update_tmp_rowset); + RPC_PREPROCESS(update_tmp_rowset, get, put); if (!request->has_rowset_meta()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "no rowset meta"; @@ -1704,7 +1697,7 @@ static bool try_fetch_and_parse_schema(Transaction* txn, RowsetMetaCloudPB& rows void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, const GetRowsetRequest* request, GetRowsetResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_rowset); + RPC_PREPROCESS(get_rowset, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -1740,11 +1733,8 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, LOG(WARNING) << msg; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); - }); + std::unique_ptr> defer_stats( + (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); TabletIndexPB idx; // Get tablet id index from kv get_tablet_idx(code, msg, txn.get(), instance_id, tablet_id, idx); @@ -1783,6 +1773,7 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, if (version_pb.pending_txn_ids_size() > 0) { DCHECK(version_pb.pending_txn_ids_size() == 1); + stats.get_counter += txn->num_get_keys(); txn.reset(); TEST_SYNC_POINT_CALLBACK("get_rowset::advance_last_pending_txn_id", &version_pb); @@ -1896,7 +1887,7 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro const GetTabletStatsRequest* request, GetTabletStatsResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_tablet_stats); + RPC_PREPROCESS(get_tablet_stats, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -1916,9 +1907,7 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); txn.reset(nullptr); }); if (!(/* idx.has_db_id() && */ idx.has_table_id() && idx.has_index_id() && @@ -2197,7 +2186,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont const UpdateDeleteBitmapRequest* request, UpdateDeleteBitmapResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(update_delete_bitmap); + RPC_PREPROCESS(update_delete_bitmap, get, put, del); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2366,9 +2355,6 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont err = txn->commit(); TEST_SYNC_POINT_CALLBACK("update_delete_bitmap:commit:err", request->initiator(), i, &err); - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); total_txn_put_keys += txn->num_put_keys(); total_txn_put_bytes += txn->put_bytes(); total_txn_size += txn->approximate_bytes(); @@ -2385,6 +2371,9 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont g_bvar_update_delete_bitmap_fail_counter << 1; return; } + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); current_key_count = 0; current_value_count = 0; TxnErrorCode err = txn_kv_->create_txn(&txn); @@ -2546,7 +2535,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control const GetDeleteBitmapRequest* request, GetDeleteBitmapResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_delete_bitmap); + RPC_PREPROCESS(get_delete_bitmap, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2592,11 +2581,8 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); - }); + std::unique_ptr> defer_stats( + (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); MetaDeleteBitmapInfo start_key_info {instance_id, tablet_id, rowset_ids[i], begin_versions[i], 0}; MetaDeleteBitmapInfo end_key_info {instance_id, tablet_id, rowset_ids[i], end_versions[i], @@ -2623,6 +2609,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control TEST_SYNC_POINT_CALLBACK("get_delete_bitmap_err", &round, &err); int64_t retry = 0; while (err == TxnErrorCode::TXN_TOO_OLD && retry < 3) { + stats.get_counter += txn->num_get_keys(); txn = nullptr; err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { @@ -2714,11 +2701,8 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); - }); + std::unique_ptr> defer_stats( + (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); TabletIndexPB idx(request->idx()); TabletStatsPB tablet_stat; internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat, @@ -2806,9 +2790,9 @@ bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std:: return false; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }); auto table_id = request->table_id(); std::stringstream ss; @@ -3003,9 +2987,9 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }); std::string lock_val; DeleteBitmapUpdateLockPB lock_info; @@ -3241,9 +3225,9 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v1( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }); auto table_id = request->table_id(); std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}); @@ -3332,9 +3316,9 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v2( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }); if (request->lock_id() == COMPACTION_DELETE_BITMAP_LOCK_ID) { std::string tablet_compaction_key = @@ -3417,9 +3401,9 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v1( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }); std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, request->table_id(), -1}); @@ -3477,7 +3461,7 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_delete_bitmap_update_lock); + RPC_PREPROCESS(get_delete_bitmap_update_lock, get, put, del); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3508,7 +3492,7 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock( google::protobuf::RpcController* controller, const RemoveDeleteBitmapUpdateLockRequest* request, RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(remove_delete_bitmap_update_lock); + RPC_PREPROCESS(remove_delete_bitmap_update_lock, get, put, del); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3539,7 +3523,7 @@ void MetaServiceImpl::remove_delete_bitmap(google::protobuf::RpcController* cont const RemoveDeleteBitmapRequest* request, RemoveDeleteBitmapResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(remove_delete_bitmap); + RPC_PREPROCESS(remove_delete_bitmap, del); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3689,7 +3673,7 @@ void MetaServiceImpl::get_schema_dict(::google::protobuf::RpcController* control const GetSchemaDictRequest* request, GetSchemaDictResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_schema_dict); + RPC_PREPROCESS(get_schema_dict, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index cce7a46add84b6..10125cf331976e 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -226,33 +226,59 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { } } -#define RPC_PREPROCESS(func_name) \ - StopWatch sw; \ - auto ctrl = static_cast(controller); \ - begin_rpc(#func_name, ctrl, request); \ - brpc::ClosureGuard closure_guard(done); \ - [[maybe_unused]] std::stringstream ss; \ - [[maybe_unused]] MetaServiceCode code = MetaServiceCode::OK; \ - [[maybe_unused]] std::unique_ptr txn; \ - [[maybe_unused]] std::string msg; \ - [[maybe_unused]] std::string instance_id; \ - [[maybe_unused]] bool drop_request = false; \ - [[maybe_unused]] KVStats stats(g_bvar_rpc_kv_##func_name##_get_counter, \ - g_bvar_rpc_kv_##func_name##_put_counter, \ - g_bvar_rpc_kv_##func_name##_del_counter); \ - std::unique_ptr> defer_status((int*)0x01, [&](int*) { \ - response->mutable_status()->set_code(code); \ - response->mutable_status()->set_msg(msg); \ - finish_rpc(#func_name, ctrl, response); \ - closure_guard.reset(nullptr); \ - if (config::use_detailed_metrics && !instance_id.empty() && !drop_request) { \ - g_bvar_ms_##func_name.put(instance_id, sw.elapsed_us()); \ - } \ - if (txn != nullptr) { \ - stats.get_counter << txn->num_get_keys(); \ - stats.put_counter << txn->num_put_keys(); \ - stats.del_counter << txn->num_del_keys(); \ - } \ +// don't use these macro it just for defer count, reduce useless variable(some rpc just need one of rw op) +// If we have to write separate code for each RPC, it would be quite troublesome +// After all, adding put, get, and del after the RPC_PREPROCESS macro is simpler than writing a long string of code +#define RPCKVCOUNTHELPER(func_name, op) \ + g_bvar_rpc_kv_##func_name##_##op##_counter << stats.op##_counter; +#define RPCKVCOUNT_0(func_name) +#define RPCKVCOUNT_1(func_name, op1) RPCKVCOUNTHELPER(func_name, op1) +#define RPCKVCOUNT_2(func_name, op1, op2) \ + RPCKVCOUNT_1(func_name, op1) RPCKVCOUNTHELPER(func_name, op2) +#define RPCKVCOUNT_3(func_name, op1, op2, op3) \ + RPCKVCOUNT_2(func_name, op1, op2) RPCKVCOUNTHELPER(func_name, op3) +#define GET_RPCKVCOUNT_MACRO(_0, _1, _2, _3, NAME, ...) NAME + +// input func_name, count type(get, put, del), make sure the counter is exist +// about defer_count: +// The C++ rule ensures that defer_count is destroyed after defer_status +// which means that these bvars will only be counted after stats has finished counting. +// why not cancle KVStats, count directly? +// 1. some RPC operations call functions and function reset txn it also need to be counted +// 2. some function such as `scan_tmp_rowset` it used by RPC(commit_txn) and non rpc +// maybe we can add a bool variable to judge weather we need count, but if have more complex situation +// `func1` used by RPC1, RPC2 and RPC3 judge it or just give func1 a pointer +#define RPC_PREPROCESS(func_name, ...) \ + StopWatch sw; \ + auto ctrl = static_cast(controller); \ + begin_rpc(#func_name, ctrl, request); \ + brpc::ClosureGuard closure_guard(done); \ + [[maybe_unused]] std::stringstream ss; \ + [[maybe_unused]] MetaServiceCode code = MetaServiceCode::OK; \ + [[maybe_unused]] std::unique_ptr txn; \ + [[maybe_unused]] std::string msg; \ + [[maybe_unused]] std::string instance_id; \ + [[maybe_unused]] bool drop_request = false; \ + [[maybe_unused]] KVStats stats; \ + std::unique_ptr> defer_count((int*)0x01, [&stats](int*) { \ + if (config::use_detailed_metrics) { \ + GET_RPCKVCOUNT_MACRO(_0, ##__VA_ARGS__, RPCKVCOUNT_3, RPCKVCOUNT_2, RPCKVCOUNT_1, \ + RPCKVCOUNT_0)(func_name, ##__VA_ARGS__) \ + } \ + }); \ + std::unique_ptr> defer_status((int*)0x01, [&](int*) { \ + response->mutable_status()->set_code(code); \ + response->mutable_status()->set_msg(msg); \ + finish_rpc(#func_name, ctrl, response); \ + closure_guard.reset(nullptr); \ + if (config::use_detailed_metrics && !instance_id.empty() && !drop_request) { \ + g_bvar_ms_##func_name.put(instance_id, sw.elapsed_us()); \ + } \ + if (txn != nullptr) { \ + stats.get_counter += txn->num_get_keys(); \ + stats.put_counter += txn->num_put_keys(); \ + stats.del_counter += txn->num_del_keys(); \ + } \ }); #define RPC_RATE_LIMIT(func_name) \ @@ -281,8 +307,7 @@ int decrypt_instance_info(InstanceInfoPB& instance, const std::string& instance_ /** * Notifies other metaservice to refresh instance */ -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, - KVStats* stats); +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id); void get_tablet_idx(MetaServiceCode& code, std::string& msg, Transaction* txn, const std::string& instance_id, int64_t tablet_id, TabletIndexPB& tablet_idx); diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index ac6408f1964722..1d6e8760d2823d 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -408,7 +408,7 @@ void MetaServiceImpl::start_tablet_job(::google::protobuf::RpcController* contro const StartTabletJobRequest* request, StartTabletJobResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(start_tablet_job); + RPC_PREPROCESS(start_tablet_job, get, put); std::string cloud_unique_id = request->cloud_unique_id(); instance_id = get_instance_id(resource_mgr_, cloud_unique_id); if (instance_id.empty()) { @@ -1475,7 +1475,7 @@ void MetaServiceImpl::finish_tablet_job(::google::protobuf::RpcController* contr const FinishTabletJobRequest* request, FinishTabletJobResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(finish_tablet_job); + RPC_PREPROCESS(finish_tablet_job, get, put, del); std::string cloud_unique_id = request->cloud_unique_id(); instance_id = get_instance_id(resource_mgr_, cloud_unique_id); if (instance_id.empty()) { diff --git a/cloud/src/meta-service/meta_service_partition.cpp b/cloud/src/meta-service/meta_service_partition.cpp index 99d544cd931628..1d8b85d5449a49 100644 --- a/cloud/src/meta-service/meta_service_partition.cpp +++ b/cloud/src/meta-service/meta_service_partition.cpp @@ -80,7 +80,7 @@ static TxnErrorCode check_recycle_key_exist(Transaction* txn, const std::string& void MetaServiceImpl::prepare_index(::google::protobuf::RpcController* controller, const IndexRequest* request, IndexResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(prepare_index); + RPC_PREPROCESS(prepare_index, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -166,7 +166,7 @@ void MetaServiceImpl::prepare_index(::google::protobuf::RpcController* controlle void MetaServiceImpl::commit_index(::google::protobuf::RpcController* controller, const IndexRequest* request, IndexResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(commit_index); + RPC_PREPROCESS(commit_index, get, put, del); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -249,7 +249,7 @@ void MetaServiceImpl::commit_index(::google::protobuf::RpcController* controller void MetaServiceImpl::drop_index(::google::protobuf::RpcController* controller, const IndexRequest* request, IndexResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(drop_index); + RPC_PREPROCESS(drop_index, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -350,7 +350,7 @@ void MetaServiceImpl::prepare_partition(::google::protobuf::RpcController* contr const PartitionRequest* request, PartitionResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(prepare_partition); + RPC_PREPROCESS(prepare_partition, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -439,7 +439,7 @@ void MetaServiceImpl::prepare_partition(::google::protobuf::RpcController* contr void MetaServiceImpl::commit_partition(::google::protobuf::RpcController* controller, const PartitionRequest* request, PartitionResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(commit_partition); + RPC_PREPROCESS(commit_partition, get, put, del); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -526,7 +526,7 @@ void MetaServiceImpl::commit_partition(::google::protobuf::RpcController* contro void MetaServiceImpl::drop_partition(::google::protobuf::RpcController* controller, const PartitionRequest* request, PartitionResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(drop_partition); + RPC_PREPROCESS(drop_partition, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -631,7 +631,7 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, return; } std::unique_ptr> defer_stats( - (int*)0x01, [&](int*) { stats.get_counter << txn->num_get_keys(); }); + (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); auto& [keys, hint, key_func] = get_check_info(request); if (keys.empty()) { *code = MetaServiceCode::INVALID_ARGUMENT; @@ -651,7 +651,7 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, *msg = "prepare and commit rpc not match, recycle key remained"; return; } else if (err == TxnErrorCode::TXN_TOO_OLD) { - stats.get_counter << txn->num_get_keys(); + stats.get_counter += txn->num_get_keys(); // separate it to several txn for rubustness txn.reset(); TxnErrorCode err = txn_kv->create_txn(&txn); @@ -677,7 +677,7 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, void MetaServiceImpl::check_kv(::google::protobuf::RpcController* controller, const CheckKVRequest* request, CheckKVResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(check_kv); + RPC_PREPROCESS(check_kv, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 4c8f9c8ca93f1c..3075bbb0c32bd3 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -203,7 +203,7 @@ void MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro const GetObjStoreInfoRequest* request, GetObjStoreInfoResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_obj_store_info); + RPC_PREPROCESS(get_obj_store_info, get); TEST_SYNC_POINT_CALLBACK("obj-store-info_sk_response", &response); TEST_SYNC_POINT_RETURN_WITH_VOID("obj-store-info_sk_response_return"); // Prepare data @@ -936,7 +936,7 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr bool use_path_style; EncryptionInfoPB encryption_info; AkSkPair cipher_ak_sk_pair; - RPC_PREPROCESS(alter_storage_vault); + RPC_PREPROCESS(alter_storage_vault, get, put, del); switch (request->op()) { case AlterObjStoreInfoRequest::ADD_S3_VAULT: case AlterObjStoreInfoRequest::DROP_S3_VAULT: { @@ -1243,7 +1243,7 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont bool use_path_style; EncryptionInfoPB encryption_info; AkSkPair cipher_ak_sk_pair; - RPC_PREPROCESS(alter_obj_store_info); + RPC_PREPROCESS(alter_obj_store_info, get, put); switch (request->op()) { case AlterObjStoreInfoRequest::ADD_OBJ_INFO: case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK: @@ -1469,7 +1469,7 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont void MetaServiceImpl::update_ak_sk(google::protobuf::RpcController* controller, const UpdateAkSkRequest* request, UpdateAkSkResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(update_ak_sk); + RPC_PREPROCESS(update_ak_sk, get, put); instance_id = request->has_instance_id() ? request->instance_id() : ""; if (instance_id.empty()) { msg = "instance id not set"; @@ -1666,7 +1666,7 @@ void MetaServiceImpl::create_instance(google::protobuf::RpcController* controlle const CreateInstanceRequest* request, CreateInstanceResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(create_instance); + RPC_PREPROCESS(create_instance, get, put); if (request->has_ram_user()) { auto& ram_user = request->ram_user(); std::string ram_user_id = ram_user.has_user_id() ? ram_user.user_id() : ""; @@ -1971,7 +1971,7 @@ void MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller if (request->op() == AlterInstanceRequest::REFRESH) return; auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_] { - notify_refresh_instance(txn_kv, instance_id, nullptr); + notify_refresh_instance(txn_kv, instance_id); }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { @@ -1983,7 +1983,7 @@ void MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller void MetaServiceImpl::get_instance(google::protobuf::RpcController* controller, const GetInstanceRequest* request, GetInstanceResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_instance); + RPC_PREPROCESS(get_instance, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2096,7 +2096,7 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, const AlterClusterRequest* request, AlterClusterResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(alter_cluster); + RPC_PREPROCESS(alter_cluster, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; instance_id = request->has_instance_id() ? request->instance_id() : ""; if (!cloud_unique_id.empty() && instance_id.empty()) { @@ -2446,10 +2446,8 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, if (code != MetaServiceCode::OK) return; - auto f = new std::function( - [instance_id = request->instance_id(), txn_kv = txn_kv_, &stats] { - notify_refresh_instance(txn_kv, instance_id, &stats); - }); + auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_, + &stats] { notify_refresh_instance(txn_kv, instance_id); }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { LOG(WARNING) << "notify refresh instance inplace, instance_id=" << request->instance_id(); @@ -2460,7 +2458,7 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, void MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller, const GetClusterRequest* request, GetClusterResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_cluster); + RPC_PREPROCESS(get_cluster, get, put); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; std::string cluster_id = request->has_cluster_id() ? request->cluster_id() : ""; std::string cluster_name = request->has_cluster_name() ? request->cluster_name() : ""; @@ -2604,7 +2602,7 @@ void MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller, void MetaServiceImpl::create_stage(::google::protobuf::RpcController* controller, const CreateStageRequest* request, CreateStageResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(create_stage); + RPC_PREPROCESS(create_stage, get, put); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2790,7 +2788,7 @@ void MetaServiceImpl::create_stage(::google::protobuf::RpcController* controller void MetaServiceImpl::get_stage(google::protobuf::RpcController* controller, const GetStageRequest* request, GetStageResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_stage); + RPC_PREPROCESS(get_stage, get); TEST_SYNC_POINT_CALLBACK("stage_sk_response", &response); TEST_SYNC_POINT_RETURN_WITH_VOID("stage_sk_response_return"); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; @@ -3179,7 +3177,7 @@ void MetaServiceImpl::drop_stage(google::protobuf::RpcController* controller, void MetaServiceImpl::get_iam(google::protobuf::RpcController* controller, const GetIamRequest* request, GetIamResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_iam); + RPC_PREPROCESS(get_iam, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3275,7 +3273,7 @@ void MetaServiceImpl::get_iam(google::protobuf::RpcController* controller, void MetaServiceImpl::alter_iam(google::protobuf::RpcController* controller, const AlterIamRequest* request, AlterIamResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(alter_iam); + RPC_PREPROCESS(alter_iam, get, put); std::string arn_id = request->has_account_id() ? request->account_id() : ""; std::string arn_ak = request->has_ak() ? request->ak() : ""; std::string arn_sk = request->has_sk() ? request->sk() : ""; @@ -3364,7 +3362,7 @@ void MetaServiceImpl::alter_ram_user(google::protobuf::RpcController* controller const AlterRamUserRequest* request, AlterRamUserResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(alter_ram_user); + RPC_PREPROCESS(alter_ram_user, get, put); instance_id = request->has_instance_id() ? request->instance_id() : ""; if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3446,7 +3444,7 @@ void MetaServiceImpl::alter_ram_user(google::protobuf::RpcController* controller void MetaServiceImpl::begin_copy(google::protobuf::RpcController* controller, const BeginCopyRequest* request, BeginCopyResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(begin_copy); + RPC_PREPROCESS(begin_copy, get, put); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3562,7 +3560,7 @@ void MetaServiceImpl::begin_copy(google::protobuf::RpcController* controller, void MetaServiceImpl::finish_copy(google::protobuf::RpcController* controller, const FinishCopyRequest* request, FinishCopyResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(finish_copy); + RPC_PREPROCESS(finish_copy, get, put, del); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3666,7 +3664,7 @@ void MetaServiceImpl::finish_copy(google::protobuf::RpcController* controller, void MetaServiceImpl::get_copy_job(google::protobuf::RpcController* controller, const GetCopyJobRequest* request, GetCopyJobResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_copy_job); + RPC_PREPROCESS(get_copy_job, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3717,7 +3715,7 @@ void MetaServiceImpl::get_copy_files(google::protobuf::RpcController* controller const GetCopyFilesRequest* request, GetCopyFilesResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_copy_files); + RPC_PREPROCESS(get_copy_files, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3780,7 +3778,7 @@ void MetaServiceImpl::filter_copy_files(google::protobuf::RpcController* control const FilterCopyFilesRequest* request, FilterCopyFilesResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(filter_copy_files); + RPC_PREPROCESS(filter_copy_files, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3831,7 +3829,7 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro const GetClusterStatusRequest* request, GetClusterStatusResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_cluster_status); + RPC_PREPROCESS(get_cluster_status, get); if (request->instance_ids().empty() && request->cloud_unique_ids().empty()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "cloud_unique_ids or instance_ids must be given, instance_ids.size: " + @@ -3881,11 +3879,8 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro LOG(WARNING) << "failed to create txn err=" << err; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); - }); + std::unique_ptr> defer_stats( + (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); err = txn->get(key, &val); LOG(INFO) << "get instance_key=" << hex(key); @@ -3930,8 +3925,7 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro msg = proto_to_json(*response); } -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, - KVStats* stats) { +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id) { LOG(INFO) << "begin notify_refresh_instance"; std::unique_ptr txn; TxnErrorCode err = txn_kv->create_txn(&txn); @@ -3939,13 +3933,6 @@ void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& i LOG(WARNING) << "failed to create txn err=" << err; return; } - std::unique_ptr> defer_status((int*)0x01, [&](int*) { - if (stats) { - stats->get_counter << txn->num_get_keys(); - stats->put_counter << txn->num_put_keys(); - stats->del_counter << txn->num_del_keys(); - } - }); std::string key = system_meta_service_registry_key(); std::string val; err = txn->get(key, &val); diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 59cab27bb23309..81754197478291 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -81,7 +81,7 @@ static void calc_table_stats(std::unordered_map& tablet_ void MetaServiceImpl::begin_txn(::google::protobuf::RpcController* controller, const BeginTxnRequest* request, BeginTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(begin_txn); + RPC_PREPROCESS(begin_txn, get, put); if (!request->has_txn_info()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid argument, missing txn info"; @@ -150,9 +150,8 @@ void MetaServiceImpl::begin_txn(::google::protobuf::RpcController* controller, msg = ss.str(); return; } - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); //2. Get txn id from version stamp txn.reset(); @@ -361,7 +360,7 @@ void MetaServiceImpl::precommit_txn(::google::protobuf::RpcController* controlle const PrecommitTxnRequest* request, PrecommitTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(precommit_txn); + RPC_PREPROCESS(precommit_txn, get, put); int64_t txn_id = request->has_txn_id() ? request->txn_id() : -1; int64_t db_id = request->has_db_id() ? request->db_id() : -1; if ((txn_id < 0 && db_id < 0)) { @@ -601,7 +600,7 @@ void MetaServiceImpl::get_rl_task_commit_attach(::google::protobuf::RpcControlle const GetRLTaskCommitAttachRequest* request, GetRLTaskCommitAttachResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_rl_task_commit_attach); + RPC_PREPROCESS(get_rl_task_commit_attach, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -669,7 +668,7 @@ void MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr const ResetRLProgressRequest* request, ResetRLProgressResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(reset_rl_progress); + RPC_PREPROCESS(reset_rl_progress, get, put, del); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -784,9 +783,7 @@ void scan_tmp_rowset( } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { if (stats) { - stats->get_counter << txn->num_get_keys(); - stats->put_counter << txn->num_put_keys(); - stats->del_counter << txn->num_del_keys(); + stats->get_counter += txn->num_get_keys(); } }); @@ -1027,9 +1024,9 @@ void commit_txn_immediately( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }); // Get txn info with db_id and txn_id @@ -1199,6 +1196,7 @@ void commit_txn_immediately( version_values.clear(); if (last_pending_txn_id > 0) { + stats.get_counter += txn->num_get_keys(); txn.reset(); TEST_SYNC_POINT_CALLBACK("commit_txn_immediately::advance_last_pending_txn_id", &last_pending_txn_id); @@ -1610,9 +1608,9 @@ void commit_txn_eventually( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }); // tablet_id -> {table/index/partition}_id @@ -1630,6 +1628,7 @@ void commit_txn_eventually( TEST_SYNC_POINT_CALLBACK("commit_txn_eventually::need_repair_tablet_idx", &need_repair_tablet_idx); if (need_repair_tablet_idx) { + stats.get_counter += txn->num_get_keys(); txn.reset(); repair_tablet_index(txn_kv, code, msg, instance_id, db_id, txn_id, tmp_rowsets_meta); if (code != MetaServiceCode::OK) { @@ -1694,6 +1693,7 @@ void commit_txn_eventually( if (last_pending_txn_id > 0) { TEST_SYNC_POINT_CALLBACK("commit_txn_eventually::advance_last_pending_txn_id", &last_pending_txn_id); + stats.get_counter += txn->num_get_keys(); txn.reset(); std::shared_ptr task = txn_lazy_committer->submit(instance_id, last_pending_txn_id); @@ -1977,9 +1977,9 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }); // Get db id with txn id @@ -2073,7 +2073,7 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* << " tmp_rowsets_meta.size()=" << tmp_rowsets_meta.size(); sub_txn_to_tmp_rowsets_meta.emplace(sub_txn_id, std::move(tmp_rowsets_meta)); } - + stats.get_counter += txn->num_get_keys(); // Create a read/write txn for guarantee consistency txn.reset(); err = txn_kv->create_txn(&txn); @@ -2561,7 +2561,7 @@ static bool force_txn_lazy_commit() { void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, const CommitTxnRequest* request, CommitTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(commit_txn); + RPC_PREPROCESS(commit_txn, get, put, del); if (!request->has_txn_id()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid argument, missing txn id"; @@ -2845,7 +2845,7 @@ static void _abort_txn(const std::string& instance_id, const AbortTxnRequest* re void MetaServiceImpl::abort_txn(::google::protobuf::RpcController* controller, const AbortTxnRequest* request, AbortTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(abort_txn); + RPC_PREPROCESS(abort_txn, get, put, del); // Get txn id int64_t txn_id = request->has_txn_id() ? request->txn_id() : -1; std::string label = request->has_label() ? request->label() : ""; @@ -2894,7 +2894,7 @@ void MetaServiceImpl::abort_txn(::google::protobuf::RpcController* controller, void MetaServiceImpl::get_txn(::google::protobuf::RpcController* controller, const GetTxnRequest* request, GetTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_txn); + RPC_PREPROCESS(get_txn, get); int64_t txn_id = request->has_txn_id() ? request->txn_id() : -1; int64_t db_id = request->has_db_id() ? request->db_id() : -1; std::string label = request->has_label() ? request->label() : ""; @@ -3022,7 +3022,7 @@ void MetaServiceImpl::get_current_max_txn_id(::google::protobuf::RpcController* const GetCurrentMaxTxnRequest* request, GetCurrentMaxTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_current_max_txn_id); + RPC_PREPROCESS(get_current_max_txn_id, get); // TODO: For auth instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { @@ -3079,7 +3079,7 @@ void MetaServiceImpl::begin_sub_txn(::google::protobuf::RpcController* controlle const BeginSubTxnRequest* request, BeginSubTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(begin_sub_txn); + RPC_PREPROCESS(begin_sub_txn, get, put, del); int64_t txn_id = request->has_txn_id() ? request->txn_id() : -1; int64_t sub_txn_num = request->has_sub_txn_num() ? request->sub_txn_num() : -1; int64_t db_id = request->has_db_id() ? request->db_id() : -1; @@ -3145,8 +3145,8 @@ void MetaServiceImpl::begin_sub_txn(::google::protobuf::RpcController* controlle msg = ss.str(); return; } - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); // 2. Get sub txn id from version stamp txn.reset(); @@ -3278,7 +3278,7 @@ void MetaServiceImpl::abort_sub_txn(::google::protobuf::RpcController* controlle const AbortSubTxnRequest* request, AbortSubTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(abort_sub_txn); + RPC_PREPROCESS(abort_sub_txn, get, put); int64_t txn_id = request->has_txn_id() ? request->txn_id() : -1; int64_t sub_txn_id = request->has_sub_txn_id() ? request->sub_txn_id() : -1; int64_t sub_txn_num = request->has_sub_txn_num() ? request->sub_txn_num() : -1; @@ -3392,7 +3392,7 @@ void MetaServiceImpl::abort_txn_with_coordinator(::google::protobuf::RpcControll const AbortTxnWithCoordinatorRequest* request, AbortTxnWithCoordinatorResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(abort_txn_with_coordinator); + RPC_PREPROCESS(abort_txn_with_coordinator, get); if (!request->has_id() || !request->has_ip() || !request->has_start_time()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid coordinate id, coordinate ip or coordinate start time."; @@ -3504,7 +3504,7 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont const CheckTxnConflictRequest* request, CheckTxnConflictResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(check_txn_conflict); + RPC_PREPROCESS(check_txn_conflict, get); if (!request->has_db_id() || !request->has_end_txn_id() || (request->table_ids_size() <= 0)) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid db id, end txn id or table_ids."; @@ -3654,9 +3654,9 @@ TxnErrorCode internal_clean_label(std::shared_ptr txn_kv, const std::stri return err; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }); err = txn->get(label_key, &label_val); @@ -3773,7 +3773,7 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control const CleanTxnLabelRequest* request, CleanTxnLabelResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(clean_txn_label); + RPC_PREPROCESS(clean_txn_label, get, put, del); if (!request->has_db_id()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "missing db id"; @@ -3813,11 +3813,8 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control << " end=" << hex(end_label_key); return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter << txn->num_get_keys(); - stats.put_counter << txn->num_put_keys(); - stats.del_counter << txn->num_del_keys(); - }); + std::unique_ptr> defer_stats( + (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); err = txn->get(begin_label_key, end_label_key, &it, snapshot, limit); if (err != TxnErrorCode::TXN_OK) { @@ -3872,7 +3869,7 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control void MetaServiceImpl::get_txn_id(::google::protobuf::RpcController* controller, const GetTxnIdRequest* request, GetTxnIdResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_txn_id); + RPC_PREPROCESS(get_txn_id, get); if (!request->has_db_id()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "missing db id"; diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index 642c7bce54327f..4e8b75c87ec6d8 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -512,7 +512,7 @@ void finish_schema_change_job( } // create_tablets -TEST(RpcKvBvarTest, DISABLED_CreateTablets) { +TEST(RpcKvBvarTest, CreateTablets) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; @@ -526,11 +526,10 @@ TEST(RpcKvBvarTest, DISABLED_CreateTablets) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_tablets_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_tablets_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_create_tablets_del_counter.get_value()); } // get_tablet -TEST(RpcKvBvarTest, DISABLED_GetTablet) { +TEST(RpcKvBvarTest, GetTablet) { std::string cloud_unique_id = "test_cloud_unique_id"; auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -551,12 +550,10 @@ TEST(RpcKvBvarTest, DISABLED_GetTablet) { LOG(INFO) << "GetTablet: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_tablet_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_tablet_del_counter.get_value()); } // get_tablet_stats -TEST(RpcKvBvarTest, DISABLED_GetTabletStats) { +TEST(RpcKvBvarTest, GetTabletStats) { std::string cloud_unique_id = "test_cloud_unique_id"; auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -573,12 +570,10 @@ TEST(RpcKvBvarTest, DISABLED_GetTabletStats) { LOG(INFO) << "GetTabletStats: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_stats_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_tablet_stats_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_tablet_stats_del_counter.get_value()); } // update_tablet -TEST(RpcKvBvarTest, DISABLED_UpdateTablet) { +TEST(RpcKvBvarTest, UpdateTablet) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; @@ -602,12 +597,11 @@ TEST(RpcKvBvarTest, DISABLED_UpdateTablet) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tablet_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tablet_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_update_tablet_del_counter.get_value()); } // update_tablet_schema // should not call update_tablet_schema -// TEST(RpcKvBvarTest, DISABLED_UpdateTabletSchema) { +// TEST(RpcKvBvarTest, UpdateTabletSchema) { // auto meta_service = get_meta_service(); // auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); // constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; @@ -636,7 +630,7 @@ TEST(RpcKvBvarTest, DISABLED_UpdateTablet) { // } // begin_txn -TEST(RpcKvBvarTest, DISABLED_BeginTxn) { +TEST(RpcKvBvarTest, BeginTxn) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 100201; @@ -656,11 +650,10 @@ TEST(RpcKvBvarTest, DISABLED_BeginTxn) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_txn_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_txn_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_begin_txn_del_counter.get_value()); } // commit_txn -TEST(RpcKvBvarTest, DISABLED_CommitTxn) { +TEST(RpcKvBvarTest, CommitTxn) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 100201; @@ -684,7 +677,7 @@ TEST(RpcKvBvarTest, DISABLED_CommitTxn) { } // precommit_txn -TEST(RpcKvBvarTest, DISABLED_PrecommitTxn) { +TEST(RpcKvBvarTest, PrecommitTxn) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); const int64_t db_id = 563413; @@ -739,11 +732,10 @@ TEST(RpcKvBvarTest, DISABLED_PrecommitTxn) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_precommit_txn_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_precommit_txn_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_precommit_txn_del_counter.get_value()); } // abort_txn -TEST(RpcKvBvarTest, DISABLED_AbortTxn) { +TEST(RpcKvBvarTest, AbortTxn) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 100201; @@ -775,7 +767,7 @@ TEST(RpcKvBvarTest, DISABLED_AbortTxn) { } // get_txn -TEST(RpcKvBvarTest, DISABLED_GetTxn) { +TEST(RpcKvBvarTest, GetTxn) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 100201; @@ -802,12 +794,10 @@ TEST(RpcKvBvarTest, DISABLED_GetTxn) { LOG(INFO) << "GetTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_txn_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_txn_del_counter.get_value()); } // get_txn_id -TEST(RpcKvBvarTest, DISABLED_GetTxnId) { +TEST(RpcKvBvarTest, GetTxnId) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 100201; @@ -835,12 +825,10 @@ TEST(RpcKvBvarTest, DISABLED_GetTxnId) { LOG(INFO) << "GetTxnId: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_id_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_txn_id_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_txn_id_del_counter.get_value()); } // prepare_rowset -TEST(RpcKvBvarTest, DISABLED_PrepareRowset) { +TEST(RpcKvBvarTest, PrepareRowset) { int64_t db_id = 100201; std::string label = "test_prepare_rowset"; auto meta_service = get_meta_service(); @@ -865,11 +853,10 @@ TEST(RpcKvBvarTest, DISABLED_PrepareRowset) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_rowset_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_rowset_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_prepare_rowset_del_counter.get_value()); } // get_rowset -TEST(RpcKvBvarTest, DISABLED_GetRowset) { +TEST(RpcKvBvarTest, GetRowset) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -891,12 +878,10 @@ TEST(RpcKvBvarTest, DISABLED_GetRowset) { LOG(INFO) << "GetRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_rowset_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_rowset_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_rowset_del_counter.get_value()); } // update_tmp_rowset -TEST(RpcKvBvarTest, DISABLED_UpdateTmpRowset) { +TEST(RpcKvBvarTest, UpdateTmpRowset) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; @@ -937,11 +922,10 @@ TEST(RpcKvBvarTest, DISABLED_UpdateTmpRowset) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tmp_rowset_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tmp_rowset_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_update_tmp_rowset_del_counter.get_value()); } // commit_rowset -TEST(RpcKvBvarTest, DISABLED_CommitRowset) { +TEST(RpcKvBvarTest, CommitRowset) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -965,7 +949,7 @@ TEST(RpcKvBvarTest, DISABLED_CommitRowset) { } // get_version -TEST(RpcKvBvarTest, DISABLED_GetVersion) { +TEST(RpcKvBvarTest, GetVersion) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -990,12 +974,10 @@ TEST(RpcKvBvarTest, DISABLED_GetVersion) { LOG(INFO) << "GetVersion: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_version_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_version_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_version_del_counter.get_value()); } // get_schema_dict -TEST(RpcKvBvarTest, DISABLED_GetSchemaDict) { +TEST(RpcKvBvarTest, GetSchemaDict) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; @@ -1023,12 +1005,10 @@ TEST(RpcKvBvarTest, DISABLED_GetSchemaDict) { LOG(INFO) << "GetSchemaDict: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_schema_dict_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_schema_dict_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_schema_dict_del_counter.get_value()); } // get_delete_bitmap_update_lock -TEST(RpcKvBvarTest, DISABLED_GetDeleteBitmapUpdateLock) { +TEST(RpcKvBvarTest, GetDeleteBitmapUpdateLock) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -1062,7 +1042,7 @@ TEST(RpcKvBvarTest, DISABLED_GetDeleteBitmapUpdateLock) { } // update_delete_bitmap -TEST(RpcKvBvarTest, DISABLED_UpdateDeleteBitmap) { +TEST(RpcKvBvarTest, UpdateDeleteBitmap) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -1101,7 +1081,7 @@ TEST(RpcKvBvarTest, DISABLED_UpdateDeleteBitmap) { } // get_delete_bitmap -TEST(RpcKvBvarTest, DISABLED_GetDeleteBitmap) { +TEST(RpcKvBvarTest, GetDeleteBitmap) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 99999; @@ -1145,12 +1125,10 @@ TEST(RpcKvBvarTest, DISABLED_GetDeleteBitmap) { LOG(INFO) << "GetDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_delete_bitmap_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_delete_bitmap_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_delete_bitmap_del_counter.get_value()); } // remove_delete_bitmap_update_lock -TEST(RpcKvBvarTest, DISABLED_RemoveDeleteBitmapUpdateLock) { +TEST(RpcKvBvarTest, RemoveDeleteBitmapUpdateLock) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -1196,7 +1174,7 @@ TEST(RpcKvBvarTest, DISABLED_RemoveDeleteBitmapUpdateLock) { } // remove_delete_bitmap -TEST(RpcKvBvarTest, DISABLED_RemoveDeleteBitmap) { +TEST(RpcKvBvarTest, RemoveDeleteBitmap) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 99999; @@ -1237,13 +1215,11 @@ TEST(RpcKvBvarTest, DISABLED_RemoveDeleteBitmap) { LOG(INFO) << "RemoveDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_remove_delete_bitmap_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_remove_delete_bitmap_put_counter.get_value()); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_remove_delete_bitmap_del_counter.get_value()); } // start_tablet_job -TEST(RpcKvBvarTest, DISABLED_StartTabletJob) { +TEST(RpcKvBvarTest, StartTabletJob) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); constexpr int64_t table_id = 10001; @@ -1264,11 +1240,10 @@ TEST(RpcKvBvarTest, DISABLED_StartTabletJob) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_start_tablet_job_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_start_tablet_job_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_start_tablet_job_del_counter.get_value()); } // finish_tablet_job -TEST(RpcKvBvarTest, DISABLED_FinishTabletJob) { +TEST(RpcKvBvarTest, FinishTabletJob) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); brpc::Controller cntl; @@ -1319,7 +1294,7 @@ TEST(RpcKvBvarTest, DISABLED_FinishTabletJob) { } // prepare_index -TEST(RpcKvBvarTest, DISABLED_PrepareIndex) { +TEST(RpcKvBvarTest, PrepareIndex) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); std::string instance_id = "test_cloud_instance_id"; @@ -1360,11 +1335,10 @@ TEST(RpcKvBvarTest, DISABLED_PrepareIndex) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_index_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_index_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_prepare_index_del_counter.get_value()); } // commit_index -TEST(RpcKvBvarTest, DISABLED_CommitIndex) { +TEST(RpcKvBvarTest, CommitIndex) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); std::string instance_id = "test_cloud_instance_id"; @@ -1411,7 +1385,7 @@ TEST(RpcKvBvarTest, DISABLED_CommitIndex) { } // drop_index -TEST(RpcKvBvarTest, DISABLED_DropIndex) { +TEST(RpcKvBvarTest, DropIndex) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 4524364; @@ -1474,11 +1448,10 @@ TEST(RpcKvBvarTest, DISABLED_DropIndex) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_index_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_index_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_drop_index_del_counter.get_value()); } // prepare_partition -TEST(RpcKvBvarTest, DISABLED_PreparePartition) { +TEST(RpcKvBvarTest, PreparePartition) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); std::string instance_id = "test_cloud_instance_id"; @@ -1521,11 +1494,10 @@ TEST(RpcKvBvarTest, DISABLED_PreparePartition) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_partition_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_partition_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_prepare_partition_del_counter.get_value()); } // commit_partition -TEST(RpcKvBvarTest, DISABLED_CommitPartition) { +TEST(RpcKvBvarTest, CommitPartition) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); std::string instance_id = "test_cloud_instance_id"; @@ -1572,7 +1544,7 @@ TEST(RpcKvBvarTest, DISABLED_CommitPartition) { } // check_kv -TEST(RpcKvBvarTest, DISABLED_CheckKv) { +TEST(RpcKvBvarTest, CheckKv) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); std::string instance_id = "test_instance"; @@ -1607,12 +1579,10 @@ TEST(RpcKvBvarTest, DISABLED_CheckKv) { LOG(INFO) << "CheckKv: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_check_kv_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_check_kv_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_check_kv_del_counter.get_value()); } // drop_partition -TEST(RpcKvBvarTest, DISABLED_DropPartition) { +TEST(RpcKvBvarTest, DropPartition) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); std::string instance_id = "test_instance"; @@ -1644,11 +1614,10 @@ TEST(RpcKvBvarTest, DISABLED_DropPartition) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_partition_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_partition_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_drop_partition_del_counter.get_value()); } // get_obj_store_info -TEST(RpcKvBvarTest, DISABLED_GetObjStoreInfo) { +TEST(RpcKvBvarTest, GetObjStoreInfo) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -1676,12 +1645,10 @@ TEST(RpcKvBvarTest, DISABLED_GetObjStoreInfo) { LOG(INFO) << "GetObjStoreInfo: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_obj_store_info_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_obj_store_info_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_obj_store_info_del_counter.get_value()); } // alter_storage_vault -TEST(RpcKvBvarTest, DISABLED_AlterStorageVault) { +TEST(RpcKvBvarTest, AlterStorageVault) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -1721,7 +1688,7 @@ TEST(RpcKvBvarTest, DISABLED_AlterStorageVault) { } // alter_obj_store_info -TEST(RpcKvBvarTest, DISABLED_AlterObjStoreInfo) { +TEST(RpcKvBvarTest, AlterObjStoreInfo) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -1777,13 +1744,12 @@ TEST(RpcKvBvarTest, DISABLED_AlterObjStoreInfo) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_obj_store_info_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_obj_store_info_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_alter_obj_store_info_del_counter.get_value()); SyncPoint::get_instance()->disable_processing(); SyncPoint::get_instance()->clear_all_call_backs(); } // update_ak_sk -TEST(RpcKvBvarTest, DISABLED_UpdateAkSk) { +TEST(RpcKvBvarTest, UpdateAkSk) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -1843,14 +1809,13 @@ TEST(RpcKvBvarTest, DISABLED_UpdateAkSk) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_ak_sk_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_ak_sk_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_update_ak_sk_del_counter.get_value()); SyncPoint::get_instance()->disable_processing(); SyncPoint::get_instance()->clear_all_call_backs(); } // create_instance -TEST(RpcKvBvarTest, DISABLED_CreateInstance) { +TEST(RpcKvBvarTest, CreateInstance) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -1893,8 +1858,6 @@ TEST(RpcKvBvarTest, DISABLED_CreateInstance) { LOG(INFO) << "CreateInstance: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_instance_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_instance_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_create_instance_del_counter.get_value()); sp->clear_all_call_backs(); sp->clear_trace(); @@ -1902,7 +1865,7 @@ TEST(RpcKvBvarTest, DISABLED_CreateInstance) { } // get_instance -TEST(RpcKvBvarTest, DISABLED_GetInstance) { +TEST(RpcKvBvarTest, GetInstance) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -1952,8 +1915,6 @@ TEST(RpcKvBvarTest, DISABLED_GetInstance) { LOG(INFO) << "GetInstance: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_instance_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_instance_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_instance_del_counter.get_value()); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -1982,12 +1943,10 @@ TEST(RpcKvBvarTest, DISABLED_GetInstance) { // LOG(INFO) << "AlterCluster: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " // << mem_kv->del_count_; // ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_cluster_get_counter.get_value()); -// ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_cluster_put_counter.get_value()); -// ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_alter_cluster_del_counter.get_value()); // } // get_cluster -TEST(RpcKvBvarTest, DISABLED_GetCluster) { +TEST(RpcKvBvarTest, GetCluster) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); InstanceKeyInfo key_info {mock_instance}; @@ -2029,11 +1988,10 @@ TEST(RpcKvBvarTest, DISABLED_GetCluster) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_cluster_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_cluster_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_cluster_del_counter.get_value()); } // create_stage -TEST(RpcKvBvarTest, DISABLED_CreateStage) { +TEST(RpcKvBvarTest, CreateStage) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -2120,14 +2078,13 @@ TEST(RpcKvBvarTest, DISABLED_CreateStage) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_stage_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_stage_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_create_stage_del_counter.get_value()); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); } // get_stage -TEST(RpcKvBvarTest, DISABLED_GetStage) { +TEST(RpcKvBvarTest, GetStage) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -2227,15 +2184,13 @@ TEST(RpcKvBvarTest, DISABLED_GetStage) { LOG(INFO) << "GetStage: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_stage_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_stage_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_stage_del_counter.get_value()); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); } // get_iam -TEST(RpcKvBvarTest, DISABLED_GetIam) { +TEST(RpcKvBvarTest, GetIam) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -2315,15 +2270,13 @@ TEST(RpcKvBvarTest, DISABLED_GetIam) { LOG(INFO) << "GetIam: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_iam_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_iam_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_iam_del_counter.get_value()); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); } // alter_iam -TEST(RpcKvBvarTest, DISABLED_AlterIam) { +TEST(RpcKvBvarTest, AlterIam) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -2361,11 +2314,10 @@ TEST(RpcKvBvarTest, DISABLED_AlterIam) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_iam_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_iam_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_alter_iam_del_counter.get_value()); } // alter_ram_user -TEST(RpcKvBvarTest, DISABLED_AlterRamUser) { +TEST(RpcKvBvarTest, AlterRamUser) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); brpc::Controller cntl; @@ -2438,14 +2390,13 @@ TEST(RpcKvBvarTest, DISABLED_AlterRamUser) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_ram_user_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_ram_user_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_alter_ram_user_del_counter.get_value()); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); } // begin_copy -TEST(RpcKvBvarTest, DISABLED_BeginCopy) { +TEST(RpcKvBvarTest, BeginCopy) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); brpc::Controller cntl; @@ -2494,14 +2445,13 @@ TEST(RpcKvBvarTest, DISABLED_BeginCopy) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_copy_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_copy_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_begin_copy_del_counter.get_value()); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); } // get_copy_job -TEST(RpcKvBvarTest, DISABLED_GetCopyJob) { +TEST(RpcKvBvarTest, GetCopyJob) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); brpc::Controller cntl; @@ -2561,15 +2511,13 @@ TEST(RpcKvBvarTest, DISABLED_GetCopyJob) { LOG(INFO) << "GetCopyJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_job_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_copy_job_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_copy_job_del_counter.get_value()); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); } // finish_copy -TEST(RpcKvBvarTest, DISABLED_FinishCopy) { +TEST(RpcKvBvarTest, FinishCopy) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); brpc::Controller cntl; @@ -2638,7 +2586,7 @@ TEST(RpcKvBvarTest, DISABLED_FinishCopy) { } // get_copy_files -TEST(RpcKvBvarTest, DISABLED_GetCopyFiles) { +TEST(RpcKvBvarTest, GetCopyFiles) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); brpc::Controller cntl; @@ -2695,15 +2643,13 @@ TEST(RpcKvBvarTest, DISABLED_GetCopyFiles) { LOG(INFO) << "GetCopyFiles: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_files_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_copy_files_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_copy_files_del_counter.get_value()); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); } // filter_copy_files -TEST(RpcKvBvarTest, DISABLED_FilterCopyFiles) { +TEST(RpcKvBvarTest, FilterCopyFiles) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); brpc::Controller cntl; @@ -2766,15 +2712,13 @@ TEST(RpcKvBvarTest, DISABLED_FilterCopyFiles) { LOG(INFO) << "FilterCopyFiles: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_filter_copy_files_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_filter_copy_files_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_filter_copy_files_del_counter.get_value()); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); } // get_cluster_status -TEST(RpcKvBvarTest, DISABLED_GetClusterStatus) { +TEST(RpcKvBvarTest, GetClusterStatus) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -2832,12 +2776,10 @@ TEST(RpcKvBvarTest, DISABLED_GetClusterStatus) { LOG(INFO) << "GetClusterStatus: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_cluster_status_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_cluster_status_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_cluster_status_del_counter.get_value()); } // get_current_max_txn_id -TEST(RpcKvBvarTest, DISABLED_GetCurrentMaxTxnId) { +TEST(RpcKvBvarTest, GetCurrentMaxTxnId) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); const int64_t db_id = 123; @@ -2882,12 +2824,10 @@ TEST(RpcKvBvarTest, DISABLED_GetCurrentMaxTxnId) { LOG(INFO) << "GetCurrentMaxTxnId: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_current_max_txn_id_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_current_max_txn_id_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_current_max_txn_id_del_counter.get_value()); } // begin_sub_txn -TEST(RpcKvBvarTest, DISABLED_BeginSubTxn) { +TEST(RpcKvBvarTest, BeginSubTxn) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 98131; @@ -2952,7 +2892,7 @@ TEST(RpcKvBvarTest, DISABLED_BeginSubTxn) { } // abort_sub_txn -TEST(RpcKvBvarTest, DISABLED_AbortSubTxn) { +TEST(RpcKvBvarTest, AbortSubTxn) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 98131; @@ -3027,11 +2967,10 @@ TEST(RpcKvBvarTest, DISABLED_AbortSubTxn) { << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_sub_txn_get_counter.get_value()); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_sub_txn_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_abort_sub_txn_del_counter.get_value()); } // abort_txn_with_coordinator -TEST(RpcKvBvarTest, DISABLED_AbortTxnWithCoordinator) { +TEST(RpcKvBvarTest, AbortTxnWithCoordinator) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); const int64_t db_id = 666; @@ -3088,12 +3027,10 @@ TEST(RpcKvBvarTest, DISABLED_AbortTxnWithCoordinator) { LOG(INFO) << "AbortTxnWithCoordinator: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_txn_with_coordinator_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_abort_txn_with_coordinator_del_counter.get_value()); } // check_txn_conflict -TEST(RpcKvBvarTest, DISABLED_CheckTxnConflict) { +TEST(RpcKvBvarTest, CheckTxnConflict) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -3149,12 +3086,10 @@ TEST(RpcKvBvarTest, DISABLED_CheckTxnConflict) { LOG(INFO) << "CheckTxnConflict: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_check_txn_conflict_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_check_txn_conflict_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_check_txn_conflict_del_counter.get_value()); } // clean_txn_label -TEST(RpcKvBvarTest, DISABLED_CleanTxnLabel) { +TEST(RpcKvBvarTest, CleanTxnLabel) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); int64_t db_id = 1987211; @@ -3192,37 +3127,4 @@ TEST(RpcKvBvarTest, DISABLED_CleanTxnLabel) { ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_clean_txn_label_put_counter.get_value()); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_clean_txn_label_del_counter.get_value()); } - -// get_rl_task_commit_attach -TEST(RpcKvBvarTest, GetRlTaskCommitAttach) { - auto meta_service = get_meta_service(); - auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); - - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; - - LOG(INFO) << "GetRlTaskCommitAttach: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ - << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_rl_task_commit_attach_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_get_rl_task_commit_attach_del_counter.get_value()); -} - -// reset_rl_progress -TEST(RpcKvBvarTest, ResetRlProgress) { - auto meta_service = get_meta_service(); - auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); - - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; - - LOG(INFO) << "ResetRlProgress: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_reset_rl_progress_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_reset_rl_progress_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_reset_rl_progress_del_counter.get_value()); -} - } // namespace doris::cloud From 2bd4d2aa03a2d2d95ab522c625a65cb3dfa23fc6 Mon Sep 17 00:00:00 2001 From: koarz Date: Wed, 18 Jun 2025 17:19:44 +0800 Subject: [PATCH 04/23] format --- cloud/src/meta-service/meta_service.cpp | 8 ++------ cloud/src/meta-service/meta_service_helper.h | 3 ++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 9f556de84455f0..8140857b1ebd7c 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -93,9 +93,7 @@ std::string get_instance_id(const std::shared_ptr& rc_mgr, std::vector nodes; std::string err = rc_mgr->get_node(cloud_unique_id, &nodes); - { - TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); - } + { TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); } std::string instance_id; if (!err.empty()) { // cache can't find cloud_unique_id, so degraded by parse cloud_unique_id @@ -292,9 +290,7 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, response->set_version(version_pb.version()); response->add_version_update_time_ms(version_pb.update_time_ms()); } - { - TEST_SYNC_POINT_CALLBACK("get_version_code", &code); - } + { TEST_SYNC_POINT_CALLBACK("get_version_code", &code); } return; } else if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { msg = "not found"; diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 10125cf331976e..480d39ca2b671e 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -263,7 +263,8 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { std::unique_ptr> defer_count((int*)0x01, [&stats](int*) { \ if (config::use_detailed_metrics) { \ GET_RPCKVCOUNT_MACRO(_0, ##__VA_ARGS__, RPCKVCOUNT_3, RPCKVCOUNT_2, RPCKVCOUNT_1, \ - RPCKVCOUNT_0)(func_name, ##__VA_ARGS__) \ + RPCKVCOUNT_0) \ + (func_name, ##__VA_ARGS__) \ } \ }); \ std::unique_ptr> defer_status((int*)0x01, [&](int*) { \ From 97f7da5f534f5b3e2ace6526dc223acde5c164e6 Mon Sep 17 00:00:00 2001 From: koarz Date: Wed, 18 Jun 2025 17:34:34 +0800 Subject: [PATCH 05/23] fix --- cloud/src/meta-service/meta_service_resource.cpp | 3 +-- cloud/test/meta_server_test.cpp | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 3075bbb0c32bd3..8ef7f6f4e3f584 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -2446,8 +2446,7 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, if (code != MetaServiceCode::OK) return; - auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_, - &stats] { notify_refresh_instance(txn_kv, instance_id); }); + auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_] { notify_refresh_instance(txn_kv, instance_id); }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { LOG(WARNING) << "notify refresh instance inplace, instance_id=" << request->instance_id(); diff --git a/cloud/test/meta_server_test.cpp b/cloud/test/meta_server_test.cpp index dcfd28285c317a..8d8f3bb79b485d 100644 --- a/cloud/test/meta_server_test.cpp +++ b/cloud/test/meta_server_test.cpp @@ -63,8 +63,7 @@ int main(int argc, char** argv) { } namespace doris::cloud { -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, - KVStats* stats); +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id); } // namespace doris::cloud TEST(MetaServerTest, FQDNRefreshInstance) { @@ -138,7 +137,7 @@ TEST(MetaServerTest, FQDNRefreshInstance) { // Refresh instance with FQDN endpoint. config::hostname = ""; - notify_refresh_instance(txn_kv, "fqdn_instance_id", nullptr); + notify_refresh_instance(txn_kv, "fqdn_instance_id"); bool refreshed = false; for (size_t i = 0; i < 100; ++i) { From 9413cb849e0be080df9195da62ba8a5d1de34fff Mon Sep 17 00:00:00 2001 From: koarz Date: Wed, 18 Jun 2025 17:46:24 +0800 Subject: [PATCH 06/23] 1 --- cloud/test/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloud/test/CMakeLists.txt b/cloud/test/CMakeLists.txt index 65c9cde561b85d..d9cc7a4886e4d6 100644 --- a/cloud/test/CMakeLists.txt +++ b/cloud/test/CMakeLists.txt @@ -43,6 +43,8 @@ add_executable(log_test log_test.cpp) add_executable(resource_test resource_test.cpp) +add_executable(rpc_kv_bvar_test rpc_kv_bvar_test.cpp mock_accessor.cpp) + add_executable(http_encode_key_test http_encode_key_test.cpp) add_executable(fdb_injection_test fdb_injection_test.cpp) @@ -84,6 +86,8 @@ target_link_libraries(log_test ${TEST_LINK_LIBS}) target_link_libraries(resource_test ${TEST_LINK_LIBS}) +target_link_libraries(rpc_kv_bvar_test ${TEST_LINK_LIBS}) + target_link_libraries(http_encode_key_test ${TEST_LINK_LIBS}) target_link_libraries(s3_accessor_test ${TEST_LINK_LIBS}) From 41f92f01636622e4060a05b441a1bc5a645fb268 Mon Sep 17 00:00:00 2001 From: koarz Date: Wed, 18 Jun 2025 17:50:28 +0800 Subject: [PATCH 07/23] format --- cloud/src/meta-service/meta_service_resource.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 8ef7f6f4e3f584..1cf7d9513f8c09 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -2446,7 +2446,9 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, if (code != MetaServiceCode::OK) return; - auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_] { notify_refresh_instance(txn_kv, instance_id); }); + auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_] { + notify_refresh_instance(txn_kv, instance_id); + }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { LOG(WARNING) << "notify refresh instance inplace, instance_id=" << request->instance_id(); From f2c7e36018fcd012781660997cc4274b9429bee4 Mon Sep 17 00:00:00 2001 From: koarz Date: Wed, 18 Jun 2025 18:49:40 +0800 Subject: [PATCH 08/23] fix --- cloud/src/common/bvars.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 3dc8859320639d..752e089de8eaea 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -17,13 +17,9 @@ #include "common/bvars.h" -<<<<<<< kv_bvar -#include -======= #include #include #include ->>>>>>> master #include #include From ff92fdf0698287df20882564685432bfbc08f47b Mon Sep 17 00:00:00 2001 From: koarz Date: Thu, 19 Jun 2025 10:07:09 +0800 Subject: [PATCH 09/23] fix --- cloud/test/rpc_kv_bvar_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index 4e8b75c87ec6d8..4346c983cf45f9 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -2807,9 +2807,9 @@ TEST(RpcKvBvarTest, GetCurrentMaxTxnId) { GetCurrentMaxTxnResponse max_txn_id_res; std::unique_ptr txn; - auto _ = mem_kv->create_txn(&txn); + auto err = mem_kv->create_txn(&txn); txn->put("schema change", "val"); - _ = txn->commit(); + err = txn->commit(); max_txn_id_req.set_cloud_unique_id(cloud_unique_id); From eaba36921d3d25baa4fcad1fb653802324ce74bf Mon Sep 17 00:00:00 2001 From: koarz Date: Thu, 19 Jun 2025 10:48:26 +0800 Subject: [PATCH 10/23] fix --- cloud/src/meta-service/meta_service_helper.h | 3 ++- cloud/src/meta-service/meta_service_resource.cpp | 15 ++++++++++++--- cloud/test/meta_server_test.cpp | 5 +++-- cloud/test/rpc_kv_bvar_test.cpp | 4 ++-- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 480d39ca2b671e..f87befa0de6697 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -308,7 +308,8 @@ int decrypt_instance_info(InstanceInfoPB& instance, const std::string& instance_ /** * Notifies other metaservice to refresh instance */ -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id); +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, + KVStats* stats); void get_tablet_idx(MetaServiceCode& code, std::string& msg, Transaction* txn, const std::string& instance_id, int64_t tablet_id, TabletIndexPB& tablet_idx); diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 1cf7d9513f8c09..55048ada7bdb35 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -28,6 +28,8 @@ #include #include +#include "common/bvars.h" +#include "common/config.h" #include "common/encryption_util.h" #include "common/logging.h" #include "common/network_util.h" @@ -1971,7 +1973,7 @@ void MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller if (request->op() == AlterInstanceRequest::REFRESH) return; auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_] { - notify_refresh_instance(txn_kv, instance_id); + notify_refresh_instance(txn_kv, instance_id, nullptr); }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { @@ -2447,7 +2449,10 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, if (code != MetaServiceCode::OK) return; auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_] { - notify_refresh_instance(txn_kv, instance_id); + // the func run with a thread, so if use macro proved stats, maybe cause stack-use-after-return error + KVStats stats; + notify_refresh_instance(txn_kv, instance_id, &stats); + g_bvar_rpc_kv_alter_cluster_get_counter << stats.get_counter; }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { @@ -3926,7 +3931,8 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro msg = proto_to_json(*response); } -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id) { +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, + KVStats* stats) { LOG(INFO) << "begin notify_refresh_instance"; std::unique_ptr txn; TxnErrorCode err = txn_kv->create_txn(&txn); @@ -3942,6 +3948,9 @@ void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& i << " err=" << err; return; } + if (config::use_detailed_metrics && stats) { + stats->get_counter++; + } std::string self_endpoint = config::hostname.empty() ? get_local_ip(config::priority_networks) : config::hostname; self_endpoint = fmt::format("{}:{}", self_endpoint, config::brpc_listen_port); diff --git a/cloud/test/meta_server_test.cpp b/cloud/test/meta_server_test.cpp index 8d8f3bb79b485d..dcfd28285c317a 100644 --- a/cloud/test/meta_server_test.cpp +++ b/cloud/test/meta_server_test.cpp @@ -63,7 +63,8 @@ int main(int argc, char** argv) { } namespace doris::cloud { -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id); +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, + KVStats* stats); } // namespace doris::cloud TEST(MetaServerTest, FQDNRefreshInstance) { @@ -137,7 +138,7 @@ TEST(MetaServerTest, FQDNRefreshInstance) { // Refresh instance with FQDN endpoint. config::hostname = ""; - notify_refresh_instance(txn_kv, "fqdn_instance_id"); + notify_refresh_instance(txn_kv, "fqdn_instance_id", nullptr); bool refreshed = false; for (size_t i = 0; i < 100; ++i) { diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index 4346c983cf45f9..5b90789c0ca73f 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -2807,9 +2807,9 @@ TEST(RpcKvBvarTest, GetCurrentMaxTxnId) { GetCurrentMaxTxnResponse max_txn_id_res; std::unique_ptr txn; - auto err = mem_kv->create_txn(&txn); + EXPECT_EQ(mem_kv->create_txn(&txn), TxnErrorCode::TXN_OK); txn->put("schema change", "val"); - err = txn->commit(); + EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); max_txn_id_req.set_cloud_unique_id(cloud_unique_id); From b665bf0d8806bb72970860510b004c2303d19fde Mon Sep 17 00:00:00 2001 From: koarz Date: Thu, 19 Jun 2025 17:50:34 +0800 Subject: [PATCH 11/23] fix --- cloud/test/rpc_kv_bvar_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index 5b90789c0ca73f..3c9b535eb14b87 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -48,7 +48,7 @@ #include "resource-manager/resource_manager.h" int main(int argc, char** argv) { - const std::string conf_file = "/mnt/disk2/lihao/doris/cloud/conf/doris_cloud.conf"; + const std::string conf_file = "doris_cloud.conf"; if (!doris::cloud::config::init(conf_file.c_str(), true)) { std::cerr << "failed to init config file, conf=" << conf_file << std::endl; return -1; From 0ae410a999aa66a6952289239aa8da968d8be65f Mon Sep 17 00:00:00 2001 From: koarz Date: Thu, 19 Jun 2025 18:53:26 +0800 Subject: [PATCH 12/23] fix --- cloud/src/meta-service/meta_service.cpp | 90 ++++++++++++------- .../meta-service/meta_service_partition.cpp | 7 +- .../meta-service/meta_service_resource.cpp | 4 +- cloud/src/meta-service/meta_service_txn.cpp | 39 +++++--- 4 files changed, 90 insertions(+), 50 deletions(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 8140857b1ebd7c..73d28b338bf633 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -93,7 +93,9 @@ std::string get_instance_id(const std::shared_ptr& rc_mgr, std::vector nodes; std::string err = rc_mgr->get_node(cloud_unique_id, &nodes); - { TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); } + { + TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); + } std::string instance_id; if (!err.empty()) { // cache can't find cloud_unique_id, so degraded by parse cloud_unique_id @@ -290,7 +292,9 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, response->set_version(version_pb.version()); response->add_version_update_time_ms(version_pb.update_time_ms()); } - { TEST_SYNC_POINT_CALLBACK("get_version_code", &code); } + { + TEST_SYNC_POINT_CALLBACK("get_version_code", &code); + } return; } else if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { msg = "not found"; @@ -361,8 +365,11 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr code = cast_as(err); break; } - std::unique_ptr> defer_stats( - (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + } + }); for (size_t i = response->versions_size(); i < num_acquired; i += BATCH_SIZE) { size_t limit = (i + BATCH_SIZE < num_acquired) ? i + BATCH_SIZE : num_acquired; version_keys.clear(); @@ -461,8 +468,10 @@ void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + } }); std::string rs_key, rs_val; @@ -601,8 +610,6 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll msg = fmt::format("failed to create txn"); return; } - std::unique_ptr> defer_stats( - (int*)0x01, [&](int*) { stats.get_counter += txn0->num_get_keys(); }); InstanceKeyInfo key_info {instance_id}; std::string key; @@ -618,7 +625,7 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll msg = ss.str(); return; } - + stats.get_counter++; if (!instance.ParseFromString(val)) { code = MetaServiceCode::PROTOBUF_PARSE_ERR; msg = "failed to parse InstanceInfoPB"; @@ -1729,8 +1736,11 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, LOG(WARNING) << msg; return; } - std::unique_ptr> defer_stats( - (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + } + }); TabletIndexPB idx; // Get tablet id index from kv get_tablet_idx(code, msg, txn.get(), instance_id, tablet_id, idx); @@ -1903,7 +1913,9 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + } txn.reset(nullptr); }); if (!(/* idx.has_db_id() && */ idx.has_table_id() && idx.has_index_id() && @@ -2577,8 +2589,11 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats( - (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + } + }); MetaDeleteBitmapInfo start_key_info {instance_id, tablet_id, rowset_ids[i], begin_versions[i], 0}; MetaDeleteBitmapInfo end_key_info {instance_id, tablet_id, rowset_ids[i], end_versions[i], @@ -2697,8 +2712,11 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats( - (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + } + }); TabletIndexPB idx(request->idx()); TabletStatsPB tablet_stat; internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat, @@ -2786,9 +2804,11 @@ bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std:: return false; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + } }); auto table_id = request->table_id(); std::stringstream ss; @@ -2983,9 +3003,11 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + } }); std::string lock_val; DeleteBitmapUpdateLockPB lock_info; @@ -3221,9 +3243,11 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v1( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + } }); auto table_id = request->table_id(); std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}); @@ -3312,9 +3336,11 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v2( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + } }); if (request->lock_id() == COMPACTION_DELETE_BITMAP_LOCK_ID) { std::string tablet_compaction_key = @@ -3397,9 +3423,11 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v1( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + } }); std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, request->table_id(), -1}); diff --git a/cloud/src/meta-service/meta_service_partition.cpp b/cloud/src/meta-service/meta_service_partition.cpp index 1d8b85d5449a49..3a163ce8f281bc 100644 --- a/cloud/src/meta-service/meta_service_partition.cpp +++ b/cloud/src/meta-service/meta_service_partition.cpp @@ -630,8 +630,11 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, *msg = "failed to create txn"; return; } - std::unique_ptr> defer_stats( - (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + } + }); auto& [keys, hint, key_func] = get_check_info(request); if (keys.empty()) { *code = MetaServiceCode::INVALID_ARGUMENT; diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 55048ada7bdb35..bcb91f60e43fd4 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -3885,8 +3885,6 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro LOG(WARNING) << "failed to create txn err=" << err; return; } - std::unique_ptr> defer_stats( - (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); err = txn->get(key, &val); LOG(INFO) << "get instance_key=" << hex(key); @@ -3894,7 +3892,7 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro LOG(WARNING) << "failed to get instance, instance_id=" << instance_id << " err=" << err; return; } - + stats.get_counter++; InstanceInfoPB instance; if (!instance.ParseFromString(val)) { LOG(WARNING) << "failed to parse InstanceInfoPB"; diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 81754197478291..6ba27a31787c31 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -1024,9 +1024,11 @@ void commit_txn_immediately( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + } }); // Get txn info with db_id and txn_id @@ -1608,9 +1610,11 @@ void commit_txn_eventually( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + } }); // tablet_id -> {table/index/partition}_id @@ -1977,9 +1981,11 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + } }); // Get db id with txn id @@ -3654,9 +3660,11 @@ TxnErrorCode internal_clean_label(std::shared_ptr txn_kv, const std::stri return err; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + } }); err = txn->get(label_key, &label_val); @@ -3813,8 +3821,11 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control << " end=" << hex(end_label_key); return; } - std::unique_ptr> defer_stats( - (int*)0x01, [&](int*) { stats.get_counter += txn->num_get_keys(); }); + std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + if (txn != nullptr) { + stats.get_counter += txn->num_get_keys(); + } + }); err = txn->get(begin_label_key, end_label_key, &it, snapshot, limit); if (err != TxnErrorCode::TXN_OK) { From 9d7dd668d55139786e3e78c5fc2b08e488f6d563 Mon Sep 17 00:00:00 2001 From: koarz Date: Thu, 19 Jun 2025 18:56:05 +0800 Subject: [PATCH 13/23] format --- cloud/src/meta-service/meta_service.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 73d28b338bf633..d3309f88a0d8c1 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -93,9 +93,7 @@ std::string get_instance_id(const std::shared_ptr& rc_mgr, std::vector nodes; std::string err = rc_mgr->get_node(cloud_unique_id, &nodes); - { - TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); - } + { TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); } std::string instance_id; if (!err.empty()) { // cache can't find cloud_unique_id, so degraded by parse cloud_unique_id @@ -292,9 +290,7 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, response->set_version(version_pb.version()); response->add_version_update_time_ms(version_pb.update_time_ms()); } - { - TEST_SYNC_POINT_CALLBACK("get_version_code", &code); - } + { TEST_SYNC_POINT_CALLBACK("get_version_code", &code); } return; } else if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { msg = "not found"; From 3e84a0435b74241eca3c27fd0e51495b8ce76a3a Mon Sep 17 00:00:00 2001 From: koarz Date: Fri, 20 Jun 2025 15:00:36 +0800 Subject: [PATCH 14/23] use mbvar --- cloud/src/common/bvars.cpp | 214 ++++++++-------- cloud/src/common/bvars.h | 218 ++++++++-------- cloud/src/meta-service/meta_service.cpp | 1 + cloud/src/meta-service/meta_service_helper.h | 6 +- .../meta-service/meta_service_resource.cpp | 19 +- cloud/src/meta-service/meta_service_txn.cpp | 2 +- cloud/test/rpc_kv_bvar_test.cpp | 236 +++++++++--------- 7 files changed, 357 insertions(+), 339 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 752e089de8eaea..73c326700027c7 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -201,169 +201,169 @@ BvarStatusWithTag g_bvar_max_rowsets_with_useless_delete_bitmap_version // rpc kv rw count // get_rowset -bvar::Adder g_bvar_rpc_kv_get_rowset_get_counter("rpc_kv","get_rowset_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_rowset_get_counter("rpc_kv_get_rowset_get_counter",{"instance_id"}); // get_version -bvar::Adder g_bvar_rpc_kv_get_version_get_counter("rpc_kv","get_version_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_version_get_counter("rpc_kv_get_version_get_counter",{"instance_id"}); // get_schema_dict -bvar::Adder g_bvar_rpc_kv_get_schema_dict_get_counter("rpc_kv","get_schema_dict_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_schema_dict_get_counter("rpc_kv_get_schema_dict_get_counter",{"instance_id"}); // create_tablets -bvar::Adder g_bvar_rpc_kv_create_tablets_get_counter("rpc_kv","create_tablets_get_counter"); -bvar::Adder g_bvar_rpc_kv_create_tablets_put_counter("rpc_kv","create_tablets_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_create_tablets_get_counter("rpc_kv_create_tablets_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_create_tablets_put_counter("rpc_kv_create_tablets_put_counter",{"instance_id"}); // update_tablet -bvar::Adder g_bvar_rpc_kv_update_tablet_get_counter("rpc_kv","update_tablet_get_counter"); -bvar::Adder g_bvar_rpc_kv_update_tablet_put_counter("rpc_kv","update_tablet_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_get_counter("rpc_kv_update_tablet_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_put_counter("rpc_kv_update_tablet_put_counter",{"instance_id"}); // update_tablet_schema -bvar::Adder g_bvar_rpc_kv_update_tablet_schema_get_counter("rpc_kv","update_tablet_schema_get_counter"); -bvar::Adder g_bvar_rpc_kv_update_tablet_schema_put_counter("rpc_kv","update_tablet_schema_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_get_counter("rpc_kv_update_tablet_schema_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_put_counter("rpc_kv_update_tablet_schema_put_counter",{"instance_id"}); // get_tablet -bvar::Adder g_bvar_rpc_kv_get_tablet_get_counter("rpc_kv","get_tablet_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_tablet_get_counter("rpc_kv_get_tablet_get_counter",{"instance_id"}); // prepare_rowset -bvar::Adder g_bvar_rpc_kv_prepare_rowset_get_counter("rpc_kv","prepare_rowset_get_counter"); -bvar::Adder g_bvar_rpc_kv_prepare_rowset_put_counter("rpc_kv","prepare_rowset_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_get_counter("rpc_kv_prepare_rowset_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_put_counter("rpc_kv_prepare_rowset_put_counter",{"instance_id"}); // commit_rowset -bvar::Adder g_bvar_rpc_kv_commit_rowset_get_counter("rpc_kv","commit_rowset_get_counter"); -bvar::Adder g_bvar_rpc_kv_commit_rowset_put_counter("rpc_kv","commit_rowset_put_counter"); -bvar::Adder g_bvar_rpc_kv_commit_rowset_del_counter("rpc_kv","commit_rowset_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_get_counter("rpc_kv_commit_rowset_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_put_counter("rpc_kv_commit_rowset_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_del_counter("rpc_kv_commit_rowset_del_counter",{"instance_id"}); // update_tmp_rowset -bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_get_counter("rpc_kv","update_tmp_rowset_get_counter"); -bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_put_counter("rpc_kv","update_tmp_rowset_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_get_counter("rpc_kv_update_tmp_rowset_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_put_counter("rpc_kv_update_tmp_rowset_put_counter",{"instance_id"}); // get_tablet_stats -bvar::Adder g_bvar_rpc_kv_get_tablet_stats_get_counter("rpc_kv","get_tablet_stats_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_tablet_stats_get_counter("rpc_kv_get_tablet_stats_get_counter",{"instance_id"}); // update_delete_bitmap -bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_get_counter("rpc_kv","update_delete_bitmap_get_counter"); -bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_put_counter("rpc_kv","update_delete_bitmap_put_counter"); -bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_del_counter("rpc_kv","update_delete_bitmap_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_get_counter("rpc_kv_update_delete_bitmap_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_put_counter("rpc_kv_update_delete_bitmap_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_del_counter("rpc_kv_update_delete_bitmap_del_counter",{"instance_id"}); // get_delete_bitmap -bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_get_counter("rpc_kv","get_delete_bitmap_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_get_counter("rpc_kv_get_delete_bitmap_get_counter",{"instance_id"}); // get_delete_bitmap_update_lock -bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter("rpc_kv","get_delete_bitmap_update_lock_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter("rpc_kv","get_delete_bitmap_update_lock_put_counter"); -bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter("rpc_kv","get_delete_bitmap_update_lock_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter("rpc_kv_get_delete_bitmap_update_lock_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter("rpc_kv_get_delete_bitmap_update_lock_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter("rpc_kv_get_delete_bitmap_update_lock_del_counter",{"instance_id"}); // remove_delete_bitmap_update_lock -bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter("rpc_kv","remove_delete_bitmap_update_lock_get_counter"); -bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter("rpc_kv","remove_delete_bitmap_update_lock_put_counter"); -bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter("rpc_kv","remove_delete_bitmap_update_lock_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter("rpc_kv_remove_delete_bitmap_update_lock_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter("rpc_kv_remove_delete_bitmap_update_lock_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter("rpc_kv_remove_delete_bitmap_update_lock_del_counter",{"instance_id"}); // remove_delete_bitmap -bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_del_counter("rpc_kv","remove_delete_bitmap_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_del_counter("rpc_kv_remove_delete_bitmap_del_counter",{"instance_id"}); // start_tablet_job -bvar::Adder g_bvar_rpc_kv_start_tablet_job_get_counter("rpc_kv","start_tablet_job_get_counter"); -bvar::Adder g_bvar_rpc_kv_start_tablet_job_put_counter("rpc_kv","start_tablet_job_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_get_counter("rpc_kv_start_tablet_job_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_put_counter("rpc_kv_start_tablet_job_put_counter",{"instance_id"}); // finish_tablet_job -bvar::Adder g_bvar_rpc_kv_finish_tablet_job_get_counter("rpc_kv","finish_tablet_job_get_counter"); -bvar::Adder g_bvar_rpc_kv_finish_tablet_job_put_counter("rpc_kv","finish_tablet_job_put_counter"); -bvar::Adder g_bvar_rpc_kv_finish_tablet_job_del_counter("rpc_kv","finish_tablet_job_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_get_counter("rpc_kv_finish_tablet_job_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_put_counter("rpc_kv_finish_tablet_job_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_del_counter("rpc_kv_finish_tablet_job_del_counter",{"instance_id"}); // prepare_index -bvar::Adder g_bvar_rpc_kv_prepare_index_get_counter("rpc_kv","prepare_index_get_counter"); -bvar::Adder g_bvar_rpc_kv_prepare_index_put_counter("rpc_kv","prepare_index_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_prepare_index_get_counter("rpc_kv_prepare_index_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_prepare_index_put_counter("rpc_kv_prepare_index_put_counter",{"instance_id"}); // commit_index -bvar::Adder g_bvar_rpc_kv_commit_index_get_counter("rpc_kv","commit_index_get_counter"); -bvar::Adder g_bvar_rpc_kv_commit_index_put_counter("rpc_kv","commit_index_put_counter"); -bvar::Adder g_bvar_rpc_kv_commit_index_del_counter("rpc_kv","commit_index_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_commit_index_get_counter("rpc_kv_commit_index_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_index_put_counter("rpc_kv_commit_index_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_index_del_counter("rpc_kv_commit_index_del_counter",{"instance_id"}); // drop_index -bvar::Adder g_bvar_rpc_kv_drop_index_get_counter("rpc_kv","drop_index_get_counter"); -bvar::Adder g_bvar_rpc_kv_drop_index_put_counter("rpc_kv","drop_index_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_drop_index_get_counter("rpc_kv_drop_index_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_drop_index_put_counter("rpc_kv_drop_index_put_counter",{"instance_id"}); // prepare_partition -bvar::Adder g_bvar_rpc_kv_prepare_partition_get_counter("rpc_kv","prepare_partition_get_counter"); -bvar::Adder g_bvar_rpc_kv_prepare_partition_put_counter("rpc_kv","prepare_partition_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_get_counter("rpc_kv_prepare_partition_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_put_counter("rpc_kv_prepare_partition_put_counter",{"instance_id"}); // commit_partition -bvar::Adder g_bvar_rpc_kv_commit_partition_get_counter("rpc_kv","commit_partition_get_counter"); -bvar::Adder g_bvar_rpc_kv_commit_partition_put_counter("rpc_kv","commit_partition_put_counter"); -bvar::Adder g_bvar_rpc_kv_commit_partition_del_counter("rpc_kv","commit_partition_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_commit_partition_get_counter("rpc_kv_commit_partition_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_partition_put_counter("rpc_kv_commit_partition_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_partition_del_counter("rpc_kv_commit_partition_del_counter",{"instance_id"}); // drop_partition -bvar::Adder g_bvar_rpc_kv_drop_partition_get_counter("rpc_kv","drop_partition_get_counter"); -bvar::Adder g_bvar_rpc_kv_drop_partition_put_counter("rpc_kv","drop_partition_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_drop_partition_get_counter("rpc_kv_drop_partition_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_drop_partition_put_counter("rpc_kv_drop_partition_put_counter",{"instance_id"}); // check_kv -bvar::Adder g_bvar_rpc_kv_check_kv_get_counter("rpc_kv","check_kv_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_check_kv_get_counter("rpc_kv_check_kv_get_counter",{"instance_id"}); // get_obj_store_info -bvar::Adder g_bvar_rpc_kv_get_obj_store_info_get_counter("rpc_kv","get_obj_store_info_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_obj_store_info_get_counter("rpc_kv_get_obj_store_info_get_counter",{"instance_id"}); // alter_storage_vault -bvar::Adder g_bvar_rpc_kv_alter_storage_vault_get_counter("rpc_kv","alter_storage_vault_get_counter"); -bvar::Adder g_bvar_rpc_kv_alter_storage_vault_put_counter("rpc_kv","alter_storage_vault_put_counter"); -bvar::Adder g_bvar_rpc_kv_alter_storage_vault_del_counter("rpc_kv","alter_storage_vault_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_get_counter("rpc_kv_alter_storage_vault_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_put_counter("rpc_kv_alter_storage_vault_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_del_counter("rpc_kv_alter_storage_vault_del_counter",{"instance_id"}); // alter_obj_store_info -bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_get_counter("rpc_kv","alter_obj_store_info_get_counter"); -bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_put_counter("rpc_kv","alter_obj_store_info_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_get_counter("rpc_kv_alter_obj_store_info_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_put_counter("rpc_kv_alter_obj_store_info_put_counter",{"instance_id"}); // update_ak_sk -bvar::Adder g_bvar_rpc_kv_update_ak_sk_get_counter("rpc_kv","update_ak_sk_get_counter"); -bvar::Adder g_bvar_rpc_kv_update_ak_sk_put_counter("rpc_kv","update_ak_sk_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_get_counter("rpc_kv_update_ak_sk_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_put_counter("rpc_kv_update_ak_sk_put_counter",{"instance_id"}); // create_instance -bvar::Adder g_bvar_rpc_kv_create_instance_get_counter("rpc_kv","create_instance_get_counter"); -bvar::Adder g_bvar_rpc_kv_create_instance_put_counter("rpc_kv","create_instance_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_create_instance_get_counter("rpc_kv_create_instance_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_create_instance_put_counter("rpc_kv_create_instance_put_counter",{"instance_id"}); // get_instance -bvar::Adder g_bvar_rpc_kv_get_instance_get_counter("rpc_kv","get_instance_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_instance_get_counter("rpc_kv_get_instance_get_counter",{"instance_id"}); // alter_cluster -bvar::Adder g_bvar_rpc_kv_alter_cluster_get_counter("rpc_kv","alter_cluster_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_alter_cluster_get_counter("rpc_kv_alter_cluster_get_counter",{"instance_id"}); // get_cluster -bvar::Adder g_bvar_rpc_kv_get_cluster_get_counter("rpc_kv","get_cluster_get_counter"); -bvar::Adder g_bvar_rpc_kv_get_cluster_put_counter("rpc_kv","get_cluster_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_cluster_get_counter("rpc_kv_get_cluster_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_cluster_put_counter("rpc_kv_get_cluster_put_counter",{"instance_id"}); // create_stage -bvar::Adder g_bvar_rpc_kv_create_stage_get_counter("rpc_kv","create_stage_get_counter"); -bvar::Adder g_bvar_rpc_kv_create_stage_put_counter("rpc_kv","create_stage_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_create_stage_get_counter("rpc_kv_create_stage_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_create_stage_put_counter("rpc_kv_create_stage_put_counter",{"instance_id"}); // get_stage -bvar::Adder g_bvar_rpc_kv_get_stage_get_counter("rpc_kv","get_stage_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_stage_get_counter("rpc_kv_get_stage_get_counter",{"instance_id"}); // get_iam -bvar::Adder g_bvar_rpc_kv_get_iam_get_counter("rpc_kv","get_iam_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_iam_get_counter("rpc_kv_get_iam_get_counter",{"instance_id"}); // alter_iam -bvar::Adder g_bvar_rpc_kv_alter_iam_get_counter("rpc_kv","alter_iam_get_counter"); -bvar::Adder g_bvar_rpc_kv_alter_iam_put_counter("rpc_kv","alter_iam_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_alter_iam_get_counter("rpc_kv_alter_iam_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_iam_put_counter("rpc_kv_alter_iam_put_counter",{"instance_id"}); // alter_ram_user -bvar::Adder g_bvar_rpc_kv_alter_ram_user_get_counter("rpc_kv","alter_ram_user_get_counter"); -bvar::Adder g_bvar_rpc_kv_alter_ram_user_put_counter("rpc_kv","alter_ram_user_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_get_counter("rpc_kv_alter_ram_user_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_put_counter("rpc_kv_alter_ram_user_put_counter",{"instance_id"}); // begin_copy -bvar::Adder g_bvar_rpc_kv_begin_copy_get_counter("rpc_kv","begin_copy_get_counter"); -bvar::Adder g_bvar_rpc_kv_begin_copy_put_counter("rpc_kv","begin_copy_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_begin_copy_get_counter("rpc_kv_begin_copy_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_copy_put_counter("rpc_kv_begin_copy_put_counter",{"instance_id"}); // finish_copy -bvar::Adder g_bvar_rpc_kv_finish_copy_get_counter("rpc_kv","finish_copy_get_counter"); -bvar::Adder g_bvar_rpc_kv_finish_copy_put_counter("rpc_kv","finish_copy_put_counter"); -bvar::Adder g_bvar_rpc_kv_finish_copy_del_counter("rpc_kv","finish_copy_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_finish_copy_get_counter("rpc_kv_finish_copy_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_copy_put_counter("rpc_kv_finish_copy_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_copy_del_counter("rpc_kv_finish_copy_del_counter",{"instance_id"}); // get_copy_job -bvar::Adder g_bvar_rpc_kv_get_copy_job_get_counter("rpc_kv","get_copy_job_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_copy_job_get_counter("rpc_kv_get_copy_job_get_counter",{"instance_id"}); // get_copy_files -bvar::Adder g_bvar_rpc_kv_get_copy_files_get_counter("rpc_kv","get_copy_files_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_copy_files_get_counter("rpc_kv_get_copy_files_get_counter",{"instance_id"}); // filter_copy_files -bvar::Adder g_bvar_rpc_kv_filter_copy_files_get_counter("rpc_kv","filter_copy_files_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_filter_copy_files_get_counter("rpc_kv_filter_copy_files_get_counter",{"instance_id"}); // get_cluster_status -bvar::Adder g_bvar_rpc_kv_get_cluster_status_get_counter("rpc_kv","get_cluster_status_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_cluster_status_get_counter("rpc_kv_get_cluster_status_get_counter",{"instance_id"}); // begin_txn -bvar::Adder g_bvar_rpc_kv_begin_txn_get_counter("rpc_kv","begin_txn_get_counter"); -bvar::Adder g_bvar_rpc_kv_begin_txn_put_counter("rpc_kv","begin_txn_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_begin_txn_get_counter("rpc_kv_begin_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_txn_put_counter("rpc_kv_begin_txn_put_counter",{"instance_id"}); // precommit_txn -bvar::Adder g_bvar_rpc_kv_precommit_txn_get_counter("rpc_kv","precommit_txn_get_counter"); -bvar::Adder g_bvar_rpc_kv_precommit_txn_put_counter("rpc_kv","precommit_txn_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_get_counter("rpc_kv_precommit_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_put_counter("rpc_kv_precommit_txn_put_counter",{"instance_id"}); // get_rl_task_commit_attach -bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter("rpc_kv","get_rl_task_commit_attach_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter("rpc_kv_get_rl_task_commit_attach_get_counter",{"instance_id"}); // reset_rl_progress -bvar::Adder g_bvar_rpc_kv_reset_rl_progress_get_counter("rpc_kv","reset_rl_progress_get_counter"); -bvar::Adder g_bvar_rpc_kv_reset_rl_progress_put_counter("rpc_kv","reset_rl_progress_put_counter"); -bvar::Adder g_bvar_rpc_kv_reset_rl_progress_del_counter("rpc_kv","reset_rl_progress_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_get_counter("rpc_kv_reset_rl_progress_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_put_counter("rpc_kv_reset_rl_progress_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_del_counter("rpc_kv_reset_rl_progress_del_counter",{"instance_id"}); // commit_txn -bvar::Adder g_bvar_rpc_kv_commit_txn_get_counter("rpc_kv","commit_txn_get_counter"); -bvar::Adder g_bvar_rpc_kv_commit_txn_put_counter("rpc_kv","commit_txn_put_counter"); -bvar::Adder g_bvar_rpc_kv_commit_txn_del_counter("rpc_kv","commit_txn_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_commit_txn_get_counter("rpc_kv_commit_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_txn_put_counter("rpc_kv_commit_txn_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_txn_del_counter("rpc_kv_commit_txn_del_counter",{"instance_id"}); // abort_txn -bvar::Adder g_bvar_rpc_kv_abort_txn_get_counter("rpc_kv","abort_txn_get_counter"); -bvar::Adder g_bvar_rpc_kv_abort_txn_put_counter("rpc_kv","abort_txn_put_counter"); -bvar::Adder g_bvar_rpc_kv_abort_txn_del_counter("rpc_kv","abort_txn_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_get_counter("rpc_kv_abort_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_put_counter("rpc_kv_abort_txn_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_del_counter("rpc_kv_abort_txn_del_counter",{"instance_id"}); // get_txn -bvar::Adder g_bvar_rpc_kv_get_txn_get_counter("rpc_kv","get_txn_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_txn_get_counter("rpc_kv_get_txn_get_counter",{"instance_id"}); // get_current_max_txn_id -bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_get_counter("rpc_kv","get_current_max_txn_id_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_current_max_txn_id_get_counter("rpc_kv_get_current_max_txn_id_get_counter",{"instance_id"}); // begin_sub_txn -bvar::Adder g_bvar_rpc_kv_begin_sub_txn_get_counter("rpc_kv","begin_sub_txn_get_counter"); -bvar::Adder g_bvar_rpc_kv_begin_sub_txn_put_counter("rpc_kv","begin_sub_txn_put_counter"); -bvar::Adder g_bvar_rpc_kv_begin_sub_txn_del_counter("rpc_kv","begin_sub_txn_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_get_counter("rpc_kv_begin_sub_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_put_counter("rpc_kv_begin_sub_txn_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_del_counter("rpc_kv_begin_sub_txn_del_counter",{"instance_id"}); // abort_sub_txn -bvar::Adder g_bvar_rpc_kv_abort_sub_txn_get_counter("rpc_kv","abort_sub_txn_get_counter"); -bvar::Adder g_bvar_rpc_kv_abort_sub_txn_put_counter("rpc_kv","abort_sub_txn_put_counter"); +mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_get_counter("rpc_kv_abort_sub_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_counter("rpc_kv_abort_sub_txn_put_counter",{"instance_id"}); // abort_txn_with_coordinator -bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter("rpc_kv","abort_txn_with_coordinator_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter("rpc_kv_abort_txn_with_coordinator_get_counter",{"instance_id"}); // check_txn_conflict -bvar::Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv","check_txn_conflict_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv_check_txn_conflict_get_counter",{"instance_id"}); // clean_txn_label -bvar::Adder g_bvar_rpc_kv_clean_txn_label_get_counter("rpc_kv","clean_txn_label_get_counter"); -bvar::Adder g_bvar_rpc_kv_clean_txn_label_put_counter("rpc_kv","clean_txn_label_put_counter"); -bvar::Adder g_bvar_rpc_kv_clean_txn_label_del_counter("rpc_kv","clean_txn_label_del_counter"); +mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter("rpc_kv_clean_txn_label_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter("rpc_kv_clean_txn_label_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter("rpc_kv_clean_txn_label_del_counter",{"instance_id"}); // get_txn_id -bvar::Adder g_bvar_rpc_kv_get_txn_id_get_counter("rpc_kv","get_txn_id_get_counter"); +mBvarInt64Adder g_bvar_rpc_kv_get_txn_id_get_counter("rpc_kv_get_txn_id_get_counter",{"instance_id"}); // clang-format on diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 1295c73a2c044b..58f56c9e6ccb14 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -150,10 +150,11 @@ class mBvarWrapper { auto get(const std::initializer_list& dim_values) { BvarType* stats = counter_.get_stats(std::list(dim_values)); + using ReturnType = decltype(stats->get_value()); if (stats) { return stats->get_value(); } - return std::declval(0); + return ReturnType {}; } private: @@ -178,6 +179,7 @@ class mBvarWrapper { }; using mBvarIntAdder = mBvarWrapper>; +using mBvarInt64Adder = mBvarWrapper>; using mBvarDoubleAdder = mBvarWrapper>; using mBvarIntRecorder = mBvarWrapper; using mBvarLatencyRecorder = mBvarWrapper; @@ -363,110 +365,110 @@ extern BvarStatusWithTag g_bvar_inverted_checker_delete_bitmaps_scanned extern BvarStatusWithTag g_bvar_max_rowsets_with_useless_delete_bitmap_version; // rpc kv -extern bvar::Adder g_bvar_rpc_kv_get_rowset_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_version_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_schema_dict_get_counter; -extern bvar::Adder g_bvar_rpc_kv_create_tablets_get_counter; -extern bvar::Adder g_bvar_rpc_kv_create_tablets_put_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tablet_get_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tablet_put_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_get_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tablet_schema_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_tablet_get_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_get_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_rowset_put_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_rowset_get_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_rowset_put_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_rowset_del_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_get_counter; -extern bvar::Adder g_bvar_rpc_kv_update_tmp_rowset_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_tablet_stats_get_counter; -extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_get_counter; -extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_put_counter; -extern bvar::Adder g_bvar_rpc_kv_update_delete_bitmap_del_counter; -extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter; -extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter; -extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter; -extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter; -extern bvar::Adder g_bvar_rpc_kv_remove_delete_bitmap_del_counter; -extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_get_counter; -extern bvar::Adder g_bvar_rpc_kv_start_tablet_job_put_counter; -extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_get_counter; -extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_put_counter; -extern bvar::Adder g_bvar_rpc_kv_finish_tablet_job_del_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_index_get_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_index_put_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_index_get_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_index_put_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_index_del_counter; -extern bvar::Adder g_bvar_rpc_kv_drop_index_get_counter; -extern bvar::Adder g_bvar_rpc_kv_drop_index_put_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_partition_get_counter; -extern bvar::Adder g_bvar_rpc_kv_prepare_partition_put_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_partition_get_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_partition_put_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_partition_del_counter; -extern bvar::Adder g_bvar_rpc_kv_drop_partition_get_counter; -extern bvar::Adder g_bvar_rpc_kv_drop_partition_put_counter; -extern bvar::Adder g_bvar_rpc_kv_check_kv_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_obj_store_info_get_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_get_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_put_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_storage_vault_del_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_get_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_obj_store_info_put_counter; -extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_get_counter; -extern bvar::Adder g_bvar_rpc_kv_update_ak_sk_put_counter; -extern bvar::Adder g_bvar_rpc_kv_create_instance_get_counter; -extern bvar::Adder g_bvar_rpc_kv_create_instance_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_instance_get_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_cluster_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_cluster_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_cluster_put_counter; -extern bvar::Adder g_bvar_rpc_kv_create_stage_get_counter; -extern bvar::Adder g_bvar_rpc_kv_create_stage_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_stage_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_iam_get_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_iam_get_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_iam_put_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_get_counter; -extern bvar::Adder g_bvar_rpc_kv_alter_ram_user_put_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_copy_get_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_copy_put_counter; -extern bvar::Adder g_bvar_rpc_kv_finish_copy_get_counter; -extern bvar::Adder g_bvar_rpc_kv_finish_copy_put_counter; -extern bvar::Adder g_bvar_rpc_kv_finish_copy_del_counter; -extern bvar::Adder g_bvar_rpc_kv_get_copy_job_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_copy_files_get_counter; -extern bvar::Adder g_bvar_rpc_kv_filter_copy_files_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_cluster_status_get_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_txn_get_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_txn_put_counter; -extern bvar::Adder g_bvar_rpc_kv_precommit_txn_get_counter; -extern bvar::Adder g_bvar_rpc_kv_precommit_txn_put_counter; -extern bvar::Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter; -extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_get_counter; -extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_put_counter; -extern bvar::Adder g_bvar_rpc_kv_reset_rl_progress_del_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_txn_get_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_txn_put_counter; -extern bvar::Adder g_bvar_rpc_kv_commit_txn_del_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_txn_get_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_txn_put_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_txn_del_counter; -extern bvar::Adder g_bvar_rpc_kv_get_txn_get_counter; -extern bvar::Adder g_bvar_rpc_kv_get_current_max_txn_id_get_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_get_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_put_counter; -extern bvar::Adder g_bvar_rpc_kv_begin_sub_txn_del_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_get_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_sub_txn_put_counter; -extern bvar::Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter; -extern bvar::Adder g_bvar_rpc_kv_check_txn_conflict_get_counter; -extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_get_counter; -extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_put_counter; -extern bvar::Adder g_bvar_rpc_kv_clean_txn_label_del_counter; -extern bvar::Adder g_bvar_rpc_kv_get_txn_id_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_rowset_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_version_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_schema_dict_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_tablets_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_tablets_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_tablet_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_tablet_stats_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_index_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_index_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_index_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_index_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_index_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_index_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_index_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_partition_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_partition_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_partition_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_partition_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_partition_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_check_kv_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_obj_store_info_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_instance_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_instance_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_instance_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_cluster_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_cluster_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_cluster_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_stage_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_stage_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_stage_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_iam_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_iam_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_iam_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_copy_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_copy_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_copy_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_copy_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_copy_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_copy_job_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_copy_files_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_filter_copy_files_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_cluster_status_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_txn_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_current_max_txn_id_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_txn_id_get_counter; diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index d3309f88a0d8c1..cd920428dcdeda 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -1912,6 +1912,7 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); } + // the txn is not a local variable, if not reset will count last res twice txn.reset(nullptr); }); if (!(/* idx.has_db_id() && */ idx.has_table_id() && idx.has_index_id() && diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index f87befa0de6697..01998255af2c6e 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -230,7 +230,7 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { // If we have to write separate code for each RPC, it would be quite troublesome // After all, adding put, get, and del after the RPC_PREPROCESS macro is simpler than writing a long string of code #define RPCKVCOUNTHELPER(func_name, op) \ - g_bvar_rpc_kv_##func_name##_##op##_counter << stats.op##_counter; + g_bvar_rpc_kv_##func_name##_##op##_counter.put({instance_id}, stats.op##_counter); #define RPCKVCOUNT_0(func_name) #define RPCKVCOUNT_1(func_name, op1) RPCKVCOUNTHELPER(func_name, op1) #define RPCKVCOUNT_2(func_name, op1, op2) \ @@ -260,8 +260,8 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { [[maybe_unused]] std::string instance_id; \ [[maybe_unused]] bool drop_request = false; \ [[maybe_unused]] KVStats stats; \ - std::unique_ptr> defer_count((int*)0x01, [&stats](int*) { \ - if (config::use_detailed_metrics) { \ + std::unique_ptr> defer_count((int*)0x01, [&](int*) { \ + if (config::use_detailed_metrics && !instance_id.empty()) { \ GET_RPCKVCOUNT_MACRO(_0, ##__VA_ARGS__, RPCKVCOUNT_3, RPCKVCOUNT_2, RPCKVCOUNT_1, \ RPCKVCOUNT_0) \ (func_name, ##__VA_ARGS__) \ diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index bcb91f60e43fd4..03cc2b1ca59467 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -2452,7 +2452,9 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, // the func run with a thread, so if use macro proved stats, maybe cause stack-use-after-return error KVStats stats; notify_refresh_instance(txn_kv, instance_id, &stats); - g_bvar_rpc_kv_alter_cluster_get_counter << stats.get_counter; + if (config::use_detailed_metrics && !instance_id.empty()) { + g_bvar_rpc_kv_alter_cluster_get_counter.put({instance_id}, stats.get_counter); + } }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { @@ -3288,9 +3290,11 @@ void MetaServiceImpl::alter_iam(google::protobuf::RpcController* controller, msg = "invalid argument"; return; } - RPC_RATE_LIMIT(alter_iam) + // for metric, give it a common instance id + instance_id = "alter_iam_instance"; + std::string key = system_meta_service_arn_info_key(); std::string val; TxnErrorCode err = txn_kv_->create_txn(&txn); @@ -3872,8 +3876,8 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro RPC_RATE_LIMIT(get_cluster_status) - auto get_clusters_info = [this, &request, &response, &has_filter, - &stats](const std::string& instance_id) { + auto get_clusters_info = [this, &request, &response, + &has_filter](const std::string& instance_id) { InstanceKeyInfo key_info {instance_id}; std::string key; std::string val; @@ -3885,6 +3889,12 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro LOG(WARNING) << "failed to create txn err=" << err; return; } + std::unique_ptr> defer_count((int*)0x01, [&](int*) { + if (config::use_detailed_metrics && txn != nullptr) { + g_bvar_rpc_kv_get_cluster_status_get_counter.put({instance_id}, + txn->num_get_keys()); + } + }); err = txn->get(key, &val); LOG(INFO) << "get instance_key=" << hex(key); @@ -3892,7 +3902,6 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro LOG(WARNING) << "failed to get instance, instance_id=" << instance_id << " err=" << err; return; } - stats.get_counter++; InstanceInfoPB instance; if (!instance.ParseFromString(val)) { LOG(WARNING) << "failed to parse InstanceInfoPB"; diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 6ba27a31787c31..a50a46fe6fc333 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -782,7 +782,7 @@ void scan_tmp_rowset( return; } std::unique_ptr> defer_stats((int*)0x01, [&](int*) { - if (stats) { + if (stats && txn) { stats->get_counter += txn->num_get_keys(); } }); diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index 3c9b535eb14b87..6c20eefeb6ecf3 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -524,8 +524,8 @@ TEST(RpcKvBvarTest, CreateTablets) { create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); LOG(INFO) << "CreateTablets: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_tablets_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_tablets_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_tablets_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_tablets_put_counter.get({mock_instance})); } // get_tablet @@ -549,7 +549,7 @@ TEST(RpcKvBvarTest, GetTablet) { meta_service->get_tablet(&cntl, &req, &resp, nullptr); LOG(INFO) << "GetTablet: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_get_counter.get({mock_instance})); } // get_tablet_stats @@ -569,7 +569,7 @@ TEST(RpcKvBvarTest, GetTabletStats) { LOG(INFO) << "GetTabletStats: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_stats_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_stats_get_counter.get({mock_instance})); } // update_tablet @@ -595,8 +595,8 @@ TEST(RpcKvBvarTest, UpdateTablet) { LOG(INFO) << "UpdateTablet: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tablet_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tablet_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tablet_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tablet_put_counter.get({mock_instance})); } // update_tablet_schema @@ -622,11 +622,11 @@ TEST(RpcKvBvarTest, UpdateTablet) { // LOG(INFO) << "UpdateTabletSchema: " << mem_kv->get_count_ << ", " // << mem_kv->put_count_ << ", " << mem_kv->del_count_; // ASSERT_EQ(mem_kv->get_count_, -// g_bvar_rpc_kv_update_tablet_schema_get_counter.get_value()); +// g_bvar_rpc_kv_update_tablet_schema_get_counter.get({mock_instance})); // ASSERT_EQ(mem_kv->put_count_, -// g_bvar_rpc_kv_update_tablet_schema_put_counter.get_value()); +// g_bvar_rpc_kv_update_tablet_schema_put_counter.get({mock_instance})); // ASSERT_EQ(mem_kv->del_count_, -// g_bvar_rpc_kv_update_tablet_schema_del_counter.get_value()); +// g_bvar_rpc_kv_update_tablet_schema_del_counter.get({mock_instance})); // } // begin_txn @@ -648,8 +648,8 @@ TEST(RpcKvBvarTest, BeginTxn) { LOG(INFO) << "BeginTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_txn_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_txn_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_txn_put_counter.get({mock_instance})); } // commit_txn @@ -671,9 +671,9 @@ TEST(RpcKvBvarTest, CommitTxn) { LOG(INFO) << "CommitTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_txn_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_txn_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_txn_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_txn_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_txn_del_counter.get({mock_instance})); } // precommit_txn @@ -730,8 +730,8 @@ TEST(RpcKvBvarTest, PrecommitTxn) { LOG(INFO) << "PrecommitTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_precommit_txn_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_precommit_txn_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_precommit_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_precommit_txn_put_counter.get({mock_instance})); } // abort_txn @@ -761,9 +761,9 @@ TEST(RpcKvBvarTest, AbortTxn) { LOG(INFO) << "AbortTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_txn_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_txn_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_abort_txn_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_txn_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_abort_txn_del_counter.get({mock_instance})); } // get_txn @@ -793,7 +793,7 @@ TEST(RpcKvBvarTest, GetTxn) { LOG(INFO) << "GetTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_get_counter.get({mock_instance})); } // get_txn_id @@ -824,7 +824,7 @@ TEST(RpcKvBvarTest, GetTxnId) { LOG(INFO) << "GetTxnId: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_id_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_id_get_counter.get({mock_instance})); } // prepare_rowset @@ -851,8 +851,8 @@ TEST(RpcKvBvarTest, PrepareRowset) { LOG(INFO) << "PrepareRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_rowset_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_rowset_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_rowset_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_rowset_put_counter.get({mock_instance})); } // get_rowset @@ -877,7 +877,7 @@ TEST(RpcKvBvarTest, GetRowset) { LOG(INFO) << "GetRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_rowset_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_rowset_get_counter.get({mock_instance})); } // update_tmp_rowset @@ -920,8 +920,8 @@ TEST(RpcKvBvarTest, UpdateTmpRowset) { LOG(INFO) << "UpdateTmpRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tmp_rowset_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tmp_rowset_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tmp_rowset_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tmp_rowset_put_counter.get({mock_instance})); } // commit_rowset @@ -943,9 +943,9 @@ TEST(RpcKvBvarTest, CommitRowset) { LOG(INFO) << "CommitRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_rowset_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_rowset_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_rowset_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_rowset_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_rowset_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_rowset_del_counter.get({mock_instance})); } // get_version @@ -973,7 +973,7 @@ TEST(RpcKvBvarTest, GetVersion) { LOG(INFO) << "GetVersion: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_version_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_version_get_counter.get({mock_instance})); } // get_schema_dict @@ -1004,7 +1004,7 @@ TEST(RpcKvBvarTest, GetSchemaDict) { LOG(INFO) << "GetSchemaDict: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_schema_dict_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_schema_dict_get_counter.get({mock_instance})); } // get_delete_bitmap_update_lock @@ -1034,11 +1034,11 @@ TEST(RpcKvBvarTest, GetDeleteBitmapUpdateLock) { LOG(INFO) << "GetDeleteBitmapUpdateLock: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, - g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter.get_value()); + g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, - g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter.get_value()); + g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, - g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter.get_value()); + g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter.get({mock_instance})); } // update_delete_bitmap @@ -1075,9 +1075,12 @@ TEST(RpcKvBvarTest, UpdateDeleteBitmap) { LOG(INFO) << "UpdateDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_delete_bitmap_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_delete_bitmap_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_update_delete_bitmap_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_update_delete_bitmap_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, + g_bvar_rpc_kv_update_delete_bitmap_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, + g_bvar_rpc_kv_update_delete_bitmap_del_counter.get({mock_instance})); } // get_delete_bitmap @@ -1124,7 +1127,7 @@ TEST(RpcKvBvarTest, GetDeleteBitmap) { LOG(INFO) << "GetDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_delete_bitmap_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_delete_bitmap_get_counter.get({mock_instance})); } // remove_delete_bitmap_update_lock @@ -1166,11 +1169,11 @@ TEST(RpcKvBvarTest, RemoveDeleteBitmapUpdateLock) { LOG(INFO) << "RemoveDeleteBitmapUpdateLock: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; ASSERT_EQ(mem_kv->get_count_, - g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter.get_value()); + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, - g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter.get_value()); + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, - g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter.get_value()); + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter.get({mock_instance})); } // remove_delete_bitmap @@ -1215,7 +1218,8 @@ TEST(RpcKvBvarTest, RemoveDeleteBitmap) { LOG(INFO) << "RemoveDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_remove_delete_bitmap_del_counter.get_value()); + ASSERT_EQ(mem_kv->del_count_, + g_bvar_rpc_kv_remove_delete_bitmap_del_counter.get({mock_instance})); } // start_tablet_job @@ -1238,8 +1242,8 @@ TEST(RpcKvBvarTest, StartTabletJob) { LOG(INFO) << "StartTabletJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_start_tablet_job_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_start_tablet_job_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_start_tablet_job_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_start_tablet_job_put_counter.get({mock_instance})); } // finish_tablet_job @@ -1288,9 +1292,9 @@ TEST(RpcKvBvarTest, FinishTabletJob) { LOG(INFO) << "FinishTabletJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_finish_tablet_job_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_finish_tablet_job_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_finish_tablet_job_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_finish_tablet_job_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_finish_tablet_job_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_finish_tablet_job_del_counter.get({mock_instance})); } // prepare_index @@ -1333,8 +1337,8 @@ TEST(RpcKvBvarTest, PrepareIndex) { LOG(INFO) << "PrepareIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_index_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_index_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_index_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_index_put_counter.get({mock_instance})); } // commit_index @@ -1379,9 +1383,9 @@ TEST(RpcKvBvarTest, CommitIndex) { LOG(INFO) << "CommitIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_index_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_index_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_index_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_index_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_index_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_index_del_counter.get({mock_instance})); } // drop_index @@ -1446,8 +1450,8 @@ TEST(RpcKvBvarTest, DropIndex) { LOG(INFO) << "DropIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_index_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_index_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_index_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_index_put_counter.get({mock_instance})); } // prepare_partition @@ -1492,8 +1496,8 @@ TEST(RpcKvBvarTest, PreparePartition) { LOG(INFO) << "PreparePartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_partition_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_partition_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_partition_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_partition_put_counter.get({mock_instance})); } // commit_partition @@ -1538,9 +1542,9 @@ TEST(RpcKvBvarTest, CommitPartition) { LOG(INFO) << "CommitPartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_partition_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_partition_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_partition_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_partition_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_partition_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_partition_del_counter.get({mock_instance})); } // check_kv @@ -1578,7 +1582,7 @@ TEST(RpcKvBvarTest, CheckKv) { LOG(INFO) << "CheckKv: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_check_kv_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_check_kv_get_counter.get({mock_instance})); } // drop_partition @@ -1612,8 +1616,8 @@ TEST(RpcKvBvarTest, DropPartition) { LOG(INFO) << "DropPartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_partition_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_partition_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_partition_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_partition_put_counter.get({mock_instance})); } // get_obj_store_info @@ -1644,7 +1648,8 @@ TEST(RpcKvBvarTest, GetObjStoreInfo) { LOG(INFO) << "GetObjStoreInfo: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_obj_store_info_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_get_obj_store_info_get_counter.get({mock_instance})); } // alter_storage_vault @@ -1682,9 +1687,12 @@ TEST(RpcKvBvarTest, AlterStorageVault) { LOG(INFO) << "AlterStorageVault: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_storage_vault_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_storage_vault_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_alter_storage_vault_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_alter_storage_vault_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, + g_bvar_rpc_kv_alter_storage_vault_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, + g_bvar_rpc_kv_alter_storage_vault_del_counter.get({mock_instance})); } // alter_obj_store_info @@ -1742,8 +1750,10 @@ TEST(RpcKvBvarTest, AlterObjStoreInfo) { LOG(INFO) << "AlterObjStoreInfo: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_obj_store_info_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_obj_store_info_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_alter_obj_store_info_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, + g_bvar_rpc_kv_alter_obj_store_info_put_counter.get({mock_instance})); SyncPoint::get_instance()->disable_processing(); SyncPoint::get_instance()->clear_all_call_backs(); } @@ -1807,8 +1817,8 @@ TEST(RpcKvBvarTest, UpdateAkSk) { LOG(INFO) << "UpdateAkSk: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_ak_sk_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_ak_sk_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_ak_sk_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_ak_sk_put_counter.get({mock_instance})); SyncPoint::get_instance()->disable_processing(); SyncPoint::get_instance()->clear_all_call_backs(); @@ -1857,7 +1867,7 @@ TEST(RpcKvBvarTest, CreateInstance) { LOG(INFO) << "CreateInstance: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_instance_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_instance_get_counter.get({mock_instance})); sp->clear_all_call_backs(); sp->clear_trace(); @@ -1914,7 +1924,7 @@ TEST(RpcKvBvarTest, GetInstance) { LOG(INFO) << "GetInstance: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_instance_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_instance_get_counter.get({mock_instance})); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -1942,7 +1952,7 @@ TEST(RpcKvBvarTest, GetInstance) { // LOG(INFO) << "AlterCluster: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " // << mem_kv->del_count_; -// ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_cluster_get_counter.get_value()); +// ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_cluster_get_counter.get({mock_instance})); // } // get_cluster @@ -1986,8 +1996,8 @@ TEST(RpcKvBvarTest, GetCluster) { LOG(INFO) << "GetCluster: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_cluster_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_cluster_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_cluster_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_cluster_put_counter.get({mock_instance})); } // create_stage @@ -2076,8 +2086,8 @@ TEST(RpcKvBvarTest, CreateStage) { LOG(INFO) << "CreateStage: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_stage_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_stage_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_stage_get_counter.get({instance_id})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_stage_put_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -2183,7 +2193,7 @@ TEST(RpcKvBvarTest, GetStage) { LOG(INFO) << "GetStage: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_stage_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_stage_get_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -2269,7 +2279,7 @@ TEST(RpcKvBvarTest, GetIam) { LOG(INFO) << "GetIam: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_iam_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_iam_get_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -2312,8 +2322,8 @@ TEST(RpcKvBvarTest, AlterIam) { LOG(INFO) << "AlterIam: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_iam_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_iam_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_iam_get_counter.get({"alter_iam_instance"})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_iam_put_counter.get({"alter_iam_instance"})); } // alter_ram_user @@ -2388,8 +2398,8 @@ TEST(RpcKvBvarTest, AlterRamUser) { LOG(INFO) << "AlterRamUser: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_ram_user_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_ram_user_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_ram_user_get_counter.get({instance_id})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_ram_user_put_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -2443,8 +2453,8 @@ TEST(RpcKvBvarTest, BeginCopy) { LOG(INFO) << "BeginCopy: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_copy_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_copy_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_copy_get_counter.get({instance_id})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_copy_put_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -2510,7 +2520,7 @@ TEST(RpcKvBvarTest, GetCopyJob) { LOG(INFO) << "GetCopyJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_job_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_job_get_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -2577,9 +2587,9 @@ TEST(RpcKvBvarTest, FinishCopy) { LOG(INFO) << "FinishCopy: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_finish_copy_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_finish_copy_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_finish_copy_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_finish_copy_get_counter.get({instance_id})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_finish_copy_put_counter.get({instance_id})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_finish_copy_del_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -2642,7 +2652,7 @@ TEST(RpcKvBvarTest, GetCopyFiles) { LOG(INFO) << "GetCopyFiles: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_files_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_files_get_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -2711,7 +2721,7 @@ TEST(RpcKvBvarTest, FilterCopyFiles) { LOG(INFO) << "FilterCopyFiles: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_filter_copy_files_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_filter_copy_files_get_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); @@ -2775,7 +2785,8 @@ TEST(RpcKvBvarTest, GetClusterStatus) { LOG(INFO) << "GetClusterStatus: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_cluster_status_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_get_cluster_status_get_counter.get({mock_instance})); } // get_current_max_txn_id @@ -2823,7 +2834,8 @@ TEST(RpcKvBvarTest, GetCurrentMaxTxnId) { LOG(INFO) << "GetCurrentMaxTxnId: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_current_max_txn_id_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_get_current_max_txn_id_get_counter.get({mock_instance})); } // begin_sub_txn @@ -2886,9 +2898,9 @@ TEST(RpcKvBvarTest, BeginSubTxn) { LOG(INFO) << "BeginSubTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_sub_txn_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_sub_txn_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_begin_sub_txn_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_sub_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_sub_txn_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_begin_sub_txn_del_counter.get({mock_instance})); } // abort_sub_txn @@ -2965,8 +2977,8 @@ TEST(RpcKvBvarTest, AbortSubTxn) { LOG(INFO) << "AbortSubTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_sub_txn_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_sub_txn_put_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_sub_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_sub_txn_put_counter.get({mock_instance})); } // abort_txn_with_coordinator @@ -3026,7 +3038,8 @@ TEST(RpcKvBvarTest, AbortTxnWithCoordinator) { LOG(INFO) << "AbortTxnWithCoordinator: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter.get({mock_instance})); } // check_txn_conflict @@ -3038,35 +3051,27 @@ TEST(RpcKvBvarTest, CheckTxnConflict) { const int64_t table_id = 777; const std::string label = "test_label"; const std::string cloud_unique_id = "test_cloud_unique_id"; - const int64_t coordinator_id = 15623; - int64_t cur_time = std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); - std::string host = "127.0.0.1:15586"; int64_t txn_id = -1; brpc::Controller begin_txn_cntl; BeginTxnRequest begin_txn_req; BeginTxnResponse begin_txn_res; TxnInfoPB txn_info_pb; - TxnCoordinatorPB coordinator; begin_txn_req.set_cloud_unique_id(cloud_unique_id); txn_info_pb.set_db_id(db_id); txn_info_pb.set_label(label); txn_info_pb.add_table_ids(table_id); txn_info_pb.set_timeout_ms(36000); - coordinator.set_id(coordinator_id); - coordinator.set_ip(host); - coordinator.set_sourcetype(::doris::cloud::TxnSourceTypePB::TXN_SOURCE_TYPE_BE); - coordinator.set_start_time(cur_time); - txn_info_pb.mutable_coordinator()->CopyFrom(coordinator); begin_txn_req.mutable_txn_info()->CopyFrom(txn_info_pb); meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), &begin_txn_req, &begin_txn_res, nullptr); + ASSERT_EQ(begin_txn_res.status().code(), MetaServiceCode::OK); + txn_id = begin_txn_res.txn_id(); + ASSERT_GT(txn_id, -1); - brpc::Controller abort_txn_conflict_cntl; + brpc::Controller check_txn_conflict_cntl; CheckTxnConflictRequest check_txn_conflict_req; CheckTxnConflictResponse check_txn_conflict_res; @@ -3085,7 +3090,8 @@ TEST(RpcKvBvarTest, CheckTxnConflict) { LOG(INFO) << "CheckTxnConflict: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_check_txn_conflict_get_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_check_txn_conflict_get_counter.get({mock_instance})); } // clean_txn_label @@ -3123,8 +3129,8 @@ TEST(RpcKvBvarTest, CleanTxnLabel) { LOG(INFO) << "CleanTxnLabel: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " << mem_kv->del_count_; - ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_clean_txn_label_get_counter.get_value()); - ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_clean_txn_label_put_counter.get_value()); - ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_clean_txn_label_del_counter.get_value()); + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_clean_txn_label_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_clean_txn_label_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_clean_txn_label_del_counter.get({mock_instance})); } } // namespace doris::cloud From 7c7bdacec80fc3023bf6702fe607094d54f9240a Mon Sep 17 00:00:00 2001 From: koarz Date: Tue, 24 Jun 2025 10:10:50 +0800 Subject: [PATCH 15/23] Update meta_service_helper.h --- cloud/src/meta-service/meta_service_helper.h | 1 - 1 file changed, 1 deletion(-) diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 84cad0ff420b3c..172e173d4104fb 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -283,7 +283,6 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { } \ }); - #define RPC_RATE_LIMIT(func_name) \ if (config::enable_rate_limit && config::use_detailed_metrics && !instance_id.empty()) { \ auto rate_limiter = rate_limiter_->get_rpc_rate_limiter(#func_name); \ From 86b42d2f35b3af099b6b9ff7e8960274ddd02b59 Mon Sep 17 00:00:00 2001 From: koarz Date: Tue, 24 Jun 2025 10:59:35 +0800 Subject: [PATCH 16/23] fix --- cloud/src/meta-service/meta_service_helper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 172e173d4104fb..8e533ca87e0768 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -281,7 +281,7 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { stats.put_counter += txn->num_put_keys(); \ stats.del_counter += txn->num_del_keys(); \ } \ - }); + }; #define RPC_RATE_LIMIT(func_name) \ if (config::enable_rate_limit && config::use_detailed_metrics && !instance_id.empty()) { \ From d89e520d71518c5890b148944076c5d6a844ebd8 Mon Sep 17 00:00:00 2001 From: koarz Date: Tue, 24 Jun 2025 11:35:00 +0800 Subject: [PATCH 17/23] format --- cloud/src/meta-service/meta_service_helper.h | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 8e533ca87e0768..256816b68bb3cf 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -242,7 +242,6 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { // input func_name, count type(get, put, del), make sure the counter is exist // about defer_count: -// The C++ rule ensures that defer_count is destroyed after defer_status // which means that these bvars will only be counted after stats has finished counting. // why not cancle KVStats, count directly? // 1. some RPC operations call functions and function reset txn it also need to be counted @@ -261,26 +260,24 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { [[maybe_unused]] std::string instance_id; \ [[maybe_unused]] bool drop_request = false; \ [[maybe_unused]] KVStats stats; \ - std::unique_ptr> defer_count((int*)0x01, [&](int*) { \ - if (config::use_detailed_metrics && !instance_id.empty()) { \ - GET_RPCKVCOUNT_MACRO(_0, ##__VA_ARGS__, RPCKVCOUNT_3, RPCKVCOUNT_2, RPCKVCOUNT_1, \ - RPCKVCOUNT_0) \ - (func_name, ##__VA_ARGS__) \ - } \ - }); \ DORIS_CLOUD_DEFER { \ response->mutable_status()->set_code(code); \ response->mutable_status()->set_msg(msg); \ finish_rpc(#func_name, ctrl, response); \ closure_guard.reset(nullptr); \ - if (config::use_detailed_metrics && !instance_id.empty() && !drop_request) { \ - g_bvar_ms_##func_name.put(instance_id, sw.elapsed_us()); \ - } \ if (txn != nullptr) { \ stats.get_counter += txn->num_get_keys(); \ stats.put_counter += txn->num_put_keys(); \ stats.del_counter += txn->num_del_keys(); \ } \ + if (config::use_detailed_metrics && !instance_id.empty()) { \ + if (!drop_request) { \ + g_bvar_ms_##func_name.put(instance_id, sw.elapsed_us()); \ + } \ + GET_RPCKVCOUNT_MACRO(_0, ##__VA_ARGS__, RPCKVCOUNT_3, RPCKVCOUNT_2, RPCKVCOUNT_1, \ + RPCKVCOUNT_0) \ + (func_name, ##__VA_ARGS__) \ + } \ }; #define RPC_RATE_LIMIT(func_name) \ From d90745155221571e0aee2c1ecc7fbf411fb1f223 Mon Sep 17 00:00:00 2001 From: koarz Date: Tue, 24 Jun 2025 15:09:07 +0800 Subject: [PATCH 18/23] Update meta_service.cpp --- cloud/src/meta-service/meta_service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index c91997939345dc..cb018c24bc56d7 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -2403,7 +2403,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont #ifdef ENABLE_INJECTION_POINT if (config::enable_update_delete_bitmap_kv_check) { - if (!check_delete_bitmap_kv_exists(code, msg, txn0, key, instance_id, tablet_id, + if (!check_delete_bitmap_kv_exists(code, msg, txn, key, instance_id, tablet_id, request->lock_id())) { return; } From 2c9dce7247c5056e380b53821b5bb2aceea6e641 Mon Sep 17 00:00:00 2001 From: koarz Date: Tue, 24 Jun 2025 18:42:58 +0800 Subject: [PATCH 19/23] use DORIS_CLOUD_DEFER --- cloud/src/meta-service/meta_service.cpp | 44 +++++++++---------- .../meta-service/meta_service_partition.cpp | 4 +- .../meta-service/meta_service_resource.cpp | 4 +- cloud/src/meta-service/meta_service_txn.cpp | 24 +++++----- cloud/test/rpc_kv_bvar_test.cpp | 27 ++++-------- 5 files changed, 47 insertions(+), 56 deletions(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index cb018c24bc56d7..1d6d164bf61f4a 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -361,11 +361,11 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr code = cast_as(err); break; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); } - }); + }; for (size_t i = response->versions_size(); i < num_acquired; i += BATCH_SIZE) { size_t limit = (i + BATCH_SIZE < num_acquired) ? i + BATCH_SIZE : num_acquired; version_keys.clear(); @@ -463,12 +463,12 @@ void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); } - }); + }; std::string rs_key, rs_val; if (has_first_rowset) { @@ -1731,11 +1731,11 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, LOG(WARNING) << msg; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); } - }); + }; TabletIndexPB idx; // Get tablet id index from kv get_tablet_idx(code, msg, txn.get(), instance_id, tablet_id, idx); @@ -1907,13 +1907,13 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro msg = fmt::format("failed to create txn, tablet_id={}", idx.tablet_id()); return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); } // the txn is not a local variable, if not reset will count last res twice txn.reset(nullptr); - }); + }; if (!(/* idx.has_db_id() && */ idx.has_table_id() && idx.has_index_id() && idx.has_partition_id() && i.has_tablet_id())) { get_tablet_idx(code, msg, txn.get(), instance_id, idx.tablet_id(), idx); @@ -2585,11 +2585,11 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); } - }); + }; MetaDeleteBitmapInfo start_key_info {instance_id, tablet_id, rowset_ids[i], begin_versions[i], 0}; MetaDeleteBitmapInfo end_key_info {instance_id, tablet_id, rowset_ids[i], end_versions[i], @@ -2708,11 +2708,11 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); } - }); + }; TabletIndexPB idx(request->idx()); TabletStatsPB tablet_stat; internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat, @@ -2799,13 +2799,13 @@ bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std:: msg = "failed to init txn"; return false; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); } - }); + }; auto table_id = request->table_id(); std::stringstream ss; if (!config::enable_batch_get_mow_tablet_stats_and_meta) { @@ -2998,13 +2998,13 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2( msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); } - }); + }; std::string lock_val; DeleteBitmapUpdateLockPB lock_info; err = txn->get(lock_key, &lock_val); @@ -3238,13 +3238,13 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v1( msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); } - }); + }; auto table_id = request->table_id(); std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}); std::string lock_val; @@ -3331,13 +3331,13 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v2( msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); } - }); + }; if (request->lock_id() == COMPACTION_DELETE_BITMAP_LOCK_ID) { std::string tablet_compaction_key = mow_tablet_compaction_key({instance_id, request->table_id(), request->initiator()}); @@ -3418,13 +3418,13 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v1( msg = "failed to init txn"; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); } - }); + }; std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, request->table_id(), -1}); std::string lock_val; diff --git a/cloud/src/meta-service/meta_service_partition.cpp b/cloud/src/meta-service/meta_service_partition.cpp index 3a163ce8f281bc..184b47e4aa56f0 100644 --- a/cloud/src/meta-service/meta_service_partition.cpp +++ b/cloud/src/meta-service/meta_service_partition.cpp @@ -630,11 +630,11 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, *msg = "failed to create txn"; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); } - }); + }; auto& [keys, hint, key_func] = get_check_info(request); if (keys.empty()) { *code = MetaServiceCode::INVALID_ARGUMENT; diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index becccfe3917123..642cc220ce7e4a 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -3879,12 +3879,12 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro LOG(WARNING) << "failed to create txn err=" << err; return; } - std::unique_ptr> defer_count((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (config::use_detailed_metrics && txn != nullptr) { g_bvar_rpc_kv_get_cluster_status_get_counter.put({instance_id}, txn->num_get_keys()); } - }); + }; err = txn->get(key, &val); LOG(INFO) << "get instance_key=" << hex(key); diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 0b6c760404abc1..025b6a3e04ae7c 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -781,11 +781,11 @@ void scan_tmp_rowset( LOG(WARNING) << msg; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (stats && txn) { stats->get_counter += txn->num_get_keys(); } - }); + }; // Get db id with txn id std::string index_val; @@ -1021,13 +1021,13 @@ void commit_txn_immediately( LOG(WARNING) << msg; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); } - }); + }; // Get txn info with db_id and txn_id std::string info_val; // Will be reused when saving updated txn @@ -1607,13 +1607,13 @@ void commit_txn_eventually( LOG(WARNING) << msg; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); } - }); + }; // tablet_id -> {table/index/partition}_id std::unordered_map tablet_ids; @@ -1978,13 +1978,13 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* LOG(WARNING) << msg; return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); } - }); + }; // Get db id with txn id std::string index_val; @@ -3656,13 +3656,13 @@ TxnErrorCode internal_clean_label(std::shared_ptr txn_kv, const std::stri << " label_key=" << hex(label_key); return err; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); } - }); + }; err = txn->get(label_key, &label_val); if (err != TxnErrorCode::TXN_OK && err != TxnErrorCode::TXN_KEY_NOT_FOUND) { @@ -3818,11 +3818,11 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control << " end=" << hex(end_label_key); return; } - std::unique_ptr> defer_stats((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (txn != nullptr) { stats.get_counter += txn->num_get_keys(); } - }); + }; err = txn->get(begin_label_key, end_label_key, &it, snapshot, limit); if (err != TxnErrorCode::TXN_OK) { diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index 6c20eefeb6ecf3..5f30610584bf2b 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -2009,8 +2009,7 @@ TEST(RpcKvBvarTest, CreateStage) { const auto* cloud_unique_id = "test_cloud_unique_id"; std::string instance_id = "stage_test_instance_id"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2102,8 +2101,7 @@ TEST(RpcKvBvarTest, GetStage) { const auto* cloud_unique_id = "test_cloud_unique_id"; std::string instance_id = "stage_test_instance_id"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2208,8 +2206,7 @@ TEST(RpcKvBvarTest, GetIam) { auto cloud_unique_id = "test_cloud_unique_id"; std::string instance_id = "get_iam_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2333,8 +2330,7 @@ TEST(RpcKvBvarTest, AlterRamUser) { brpc::Controller cntl; std::string instance_id = "alter_ram_user_instance_id"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2416,8 +2412,7 @@ TEST(RpcKvBvarTest, BeginCopy) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2471,8 +2466,7 @@ TEST(RpcKvBvarTest, GetCopyJob) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2537,8 +2531,7 @@ TEST(RpcKvBvarTest, FinishCopy) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2606,8 +2599,7 @@ TEST(RpcKvBvarTest, GetCopyFiles) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2669,8 +2661,7 @@ TEST(RpcKvBvarTest, FilterCopyFiles) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; From c156349b77dc1ce27c315406591d48d82832cf18 Mon Sep 17 00:00:00 2001 From: koarz Date: Tue, 24 Jun 2025 18:45:46 +0800 Subject: [PATCH 20/23] 1 --- cloud/src/meta-service/meta_service.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 1d6d164bf61f4a..0d383f09b4e4ac 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -1067,7 +1067,6 @@ bool check_job_existed(Transaction* txn, MetaServiceCode& code, std::string& msg * @param txn_id The transaction ID to check. * @param code Reference to the error code to be set in case of failure. * @param msg Reference to the error message to be set in case of failure. -* @param stats count rpc kv read write num * @return true if the transaction status matches the expected status, false otherwise. */ static bool check_transaction_status(TxnStatusPB expect_status, Transaction* txn, From d4613ddcb8760b69aa96e4a0b0b0c1e80a30f5d1 Mon Sep 17 00:00:00 2001 From: koarz Date: Tue, 24 Jun 2025 18:48:22 +0800 Subject: [PATCH 21/23] format --- cloud/test/rpc_kv_bvar_test.cpp | 36 ++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index 5f30610584bf2b..916d7bc29744e4 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -2009,7 +2009,9 @@ TEST(RpcKvBvarTest, CreateStage) { const auto* cloud_unique_id = "test_cloud_unique_id"; std::string instance_id = "stage_test_instance_id"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2101,7 +2103,9 @@ TEST(RpcKvBvarTest, GetStage) { const auto* cloud_unique_id = "test_cloud_unique_id"; std::string instance_id = "stage_test_instance_id"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2206,7 +2210,9 @@ TEST(RpcKvBvarTest, GetIam) { auto cloud_unique_id = "test_cloud_unique_id"; std::string instance_id = "get_iam_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2330,7 +2336,9 @@ TEST(RpcKvBvarTest, AlterRamUser) { brpc::Controller cntl; std::string instance_id = "alter_ram_user_instance_id"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2412,7 +2420,9 @@ TEST(RpcKvBvarTest, BeginCopy) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2466,7 +2476,9 @@ TEST(RpcKvBvarTest, GetCopyJob) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2531,7 +2543,9 @@ TEST(RpcKvBvarTest, FinishCopy) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2599,7 +2613,9 @@ TEST(RpcKvBvarTest, GetCopyFiles) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2661,7 +2677,9 @@ TEST(RpcKvBvarTest, FilterCopyFiles) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - DORIS_CLOUD_DEFER { SyncPoint::get_instance()->clear_all_call_backs(); }; + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; From 3efdecc5f10ac7d9e13c539b06c26d193dfda4fe Mon Sep 17 00:00:00 2001 From: koarz Date: Wed, 25 Jun 2025 15:15:10 +0800 Subject: [PATCH 22/23] fix --- cloud/src/meta-service/mem_txn_kv.cpp | 4 +- cloud/src/meta-service/meta_service.cpp | 86 +++++++++---------- .../meta-service/meta_service_partition.cpp | 5 +- .../src/meta-service/meta_service_schema.cpp | 2 - cloud/src/meta-service/meta_service_txn.cpp | 46 +++++----- cloud/src/meta-service/txn_kv.cpp | 2 +- cloud/test/rpc_kv_bvar_test.cpp | 4 - 7 files changed, 64 insertions(+), 85 deletions(-) diff --git a/cloud/src/meta-service/mem_txn_kv.cpp b/cloud/src/meta-service/mem_txn_kv.cpp index bc7b92efdb2853..ebd62fc12a9015 100644 --- a/cloud/src/meta-service/mem_txn_kv.cpp +++ b/cloud/src/meta-service/mem_txn_kv.cpp @@ -271,6 +271,8 @@ TxnErrorCode Transaction::get(std::string_view begin, std::string_view end, int limit) { TEST_SYNC_POINT_CALLBACK("memkv::Transaction::get", &limit); std::lock_guard l(lock_); + num_get_keys_++; + kv_->get_count_++; std::string begin_k(begin.data(), begin.size()); std::string end_k(end.data(), end.size()); // TODO: figure out what happen if range_get has part of unreadable_keys @@ -301,8 +303,6 @@ TxnErrorCode Transaction::inner_get(const std::string& key, std::string* val, bo return TxnErrorCode::TXN_KEY_NOT_FOUND; } } - num_get_keys_++; - kv_->get_count_++; return TxnErrorCode::TXN_OK; } diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 0d383f09b4e4ac..7de21d985b3f3d 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -93,7 +93,9 @@ std::string get_instance_id(const std::shared_ptr& rc_mgr, std::vector nodes; std::string err = rc_mgr->get_node(cloud_unique_id, &nodes); - { TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); } + { + TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); + } std::string instance_id; if (!err.empty()) { // cache can't find cloud_unique_id, so degraded by parse cloud_unique_id @@ -290,7 +292,9 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, response->set_version(version_pb.version()); response->add_version_update_time_ms(version_pb.update_time_ms()); } - { TEST_SYNC_POINT_CALLBACK("get_version_code", &code); } + { + TEST_SYNC_POINT_CALLBACK("get_version_code", &code); + } return; } else if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { msg = "not found"; @@ -362,9 +366,8 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr break; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); }; for (size_t i = response->versions_size(); i < num_acquired; i += BATCH_SIZE) { size_t limit = (i + BATCH_SIZE < num_acquired) ? i + BATCH_SIZE : num_acquired; @@ -464,10 +467,9 @@ void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); }; std::string rs_key, rs_val; @@ -613,6 +615,7 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll instance_key(key_info, &key); err = txn0->get(key, &val); + stats.get_counter++; LOG(INFO) << "get instance_key=" << hex(key); if (err != TxnErrorCode::TXN_OK) { @@ -621,7 +624,6 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll msg = ss.str(); return; } - stats.get_counter++; if (!instance.ParseFromString(val)) { code = MetaServiceCode::PROTOBUF_PARSE_ERR; msg = "failed to parse InstanceInfoPB"; @@ -1731,9 +1733,8 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); }; TabletIndexPB idx; // Get tablet id index from kv @@ -1907,9 +1908,7 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - } + stats.get_counter += txn->num_get_keys(); // the txn is not a local variable, if not reset will count last res twice txn.reset(nullptr); }; @@ -2585,9 +2584,8 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); }; MetaDeleteBitmapInfo start_key_info {instance_id, tablet_id, rowset_ids[i], begin_versions[i], 0}; @@ -2708,9 +2706,8 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); }; TabletIndexPB idx(request->idx()); TabletStatsPB tablet_stat; @@ -2799,11 +2796,10 @@ bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std:: return false; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }; auto table_id = request->table_id(); std::stringstream ss; @@ -2998,11 +2994,10 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2( return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }; std::string lock_val; DeleteBitmapUpdateLockPB lock_info; @@ -3238,11 +3233,10 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v1( return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }; auto table_id = request->table_id(); std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}); @@ -3331,11 +3325,10 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v2( return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }; if (request->lock_id() == COMPACTION_DELETE_BITMAP_LOCK_ID) { std::string tablet_compaction_key = @@ -3418,11 +3411,10 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock_v1( return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }; std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, request->table_id(), -1}); diff --git a/cloud/src/meta-service/meta_service_partition.cpp b/cloud/src/meta-service/meta_service_partition.cpp index 184b47e4aa56f0..5afb24bfe93aea 100644 --- a/cloud/src/meta-service/meta_service_partition.cpp +++ b/cloud/src/meta-service/meta_service_partition.cpp @@ -631,9 +631,8 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); }; auto& [keys, hint, key_func] = get_check_info(request); if (keys.empty()) { diff --git a/cloud/src/meta-service/meta_service_schema.cpp b/cloud/src/meta-service/meta_service_schema.cpp index 6f980e46a0f826..31c067faea4991 100644 --- a/cloud/src/meta-service/meta_service_schema.cpp +++ b/cloud/src/meta-service/meta_service_schema.cpp @@ -26,12 +26,10 @@ #include #include -#include #include #include "common/config.h" #include "common/logging.h" -#include "common/stats.h" #include "common/util.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 025b6a3e04ae7c..078c3c7ff5404d 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -150,6 +150,7 @@ void MetaServiceImpl::begin_txn(::google::protobuf::RpcController* controller, msg = ss.str(); return; } + // get count before txn reset, if not we will lose these count stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); //2. Get txn id from version stamp @@ -782,9 +783,7 @@ void scan_tmp_rowset( return; } DORIS_CLOUD_DEFER { - if (stats && txn) { - stats->get_counter += txn->num_get_keys(); - } + if (stats && txn) stats->get_counter += txn->num_get_keys(); }; // Get db id with txn id @@ -1022,11 +1021,10 @@ void commit_txn_immediately( return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }; // Get txn info with db_id and txn_id @@ -1608,11 +1606,10 @@ void commit_txn_eventually( return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }; // tablet_id -> {table/index/partition}_id @@ -1979,11 +1976,10 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }; // Get db id with txn id @@ -3657,11 +3653,10 @@ TxnErrorCode internal_clean_label(std::shared_ptr txn_kv, const std::stri return err; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); }; err = txn->get(label_key, &label_val); @@ -3819,9 +3814,8 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control return; } DORIS_CLOUD_DEFER { - if (txn != nullptr) { - stats.get_counter += txn->num_get_keys(); - } + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); }; err = txn->get(begin_label_key, end_label_key, &it, snapshot, limit); diff --git a/cloud/src/meta-service/txn_kv.cpp b/cloud/src/meta-service/txn_kv.cpp index 6d2d991adc32c5..56e5440c8fe9af 100644 --- a/cloud/src/meta-service/txn_kv.cpp +++ b/cloud/src/meta-service/txn_kv.cpp @@ -399,6 +399,7 @@ TxnErrorCode Transaction::get(std::string_view key, std::string* val, bool snaps const uint8_t* ret; int len; err = fdb_future_get_value(fut, &found, &ret, &len); + num_get_keys_++; if (err) { LOG(WARNING) << __PRETTY_FUNCTION__ @@ -408,7 +409,6 @@ TxnErrorCode Transaction::get(std::string_view key, std::string* val, bool snaps } if (!found) return TxnErrorCode::TXN_KEY_NOT_FOUND; - num_get_keys_++; *val = std::string((char*)ret, len); return TxnErrorCode::TXN_OK; } diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index 916d7bc29744e4..6f6bf9d0d201ac 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -28,11 +28,8 @@ #include #include -#include #include #include -#include -#include #include "common/config.h" #include "common/logging.h" @@ -45,7 +42,6 @@ #include "meta-service/txn_kv_error.h" #include "mock_resource_manager.h" #include "rate-limiter/rate_limiter.h" -#include "resource-manager/resource_manager.h" int main(int argc, char** argv) { const std::string conf_file = "doris_cloud.conf"; From 2cb9a85d46c04732e3a3eff3bceba1ec99747910 Mon Sep 17 00:00:00 2001 From: koarz Date: Wed, 25 Jun 2025 15:35:26 +0800 Subject: [PATCH 23/23] format --- cloud/src/meta-service/meta_service.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 7de21d985b3f3d..df32a397e39103 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -93,9 +93,7 @@ std::string get_instance_id(const std::shared_ptr& rc_mgr, std::vector nodes; std::string err = rc_mgr->get_node(cloud_unique_id, &nodes); - { - TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); - } + { TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); } std::string instance_id; if (!err.empty()) { // cache can't find cloud_unique_id, so degraded by parse cloud_unique_id @@ -292,9 +290,7 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, response->set_version(version_pb.version()); response->add_version_update_time_ms(version_pb.update_time_ms()); } - { - TEST_SYNC_POINT_CALLBACK("get_version_code", &code); - } + { TEST_SYNC_POINT_CALLBACK("get_version_code", &code); } return; } else if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { msg = "not found";