From 13b6bae766b9450b7627668e436818e38e272912 Mon Sep 17 00:00:00 2001 From: Steven Czabaniuk Date: Mon, 13 Nov 2023 12:39:09 -0600 Subject: [PATCH 1/3] Create BlockstoreRpcApiMetrics struct for tracking number of RPC calls --- ledger/src/blockstore_metrics.rs | 98 ++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/ledger/src/blockstore_metrics.rs b/ledger/src/blockstore_metrics.rs index 46cceb55b36a7c..eea1ba0ce01d0b 100644 --- a/ledger/src/blockstore_metrics.rs +++ b/ledger/src/blockstore_metrics.rs @@ -143,6 +143,104 @@ impl BlockstoreInsertionMetrics { } } +/// A metrics struct to track the number of times Blockstore RPC function are called. +#[derive(Default)] +pub(crate) struct BlockstoreRpcApiMetrics { + pub num_get_block_height: AtomicU64, + pub num_get_complete_transaction: AtomicU64, + pub num_get_confirmed_signatures_for_address: AtomicU64, + pub num_get_confirmed_signatures_for_address2: AtomicU64, + pub num_get_rooted_block: AtomicU64, + pub num_get_rooted_block_time: AtomicU64, + pub num_get_rooted_transaction: AtomicU64, + pub num_get_rooted_transaction_status: AtomicU64, + pub num_get_rooted_block_with_entries: AtomicU64, + pub num_get_transaction_status: AtomicU64, +} + +impl BlockstoreRpcApiMetrics { + pub fn report(&self) { + let num_get_block_height = self.num_get_block_height.swap(0, Ordering::Relaxed); + let num_get_complete_transaction = + self.num_get_complete_transaction.swap(0, Ordering::Relaxed); + let num_get_confirmed_signatures_for_address = self + .num_get_confirmed_signatures_for_address + .swap(0, Ordering::Relaxed); + let num_get_confirmed_signatures_for_address2 = self + .num_get_confirmed_signatures_for_address2 + .swap(0, Ordering::Relaxed); + let num_get_rooted_block = self.num_get_rooted_block.swap(0, Ordering::Relaxed); + let num_get_rooted_block_time = self.num_get_rooted_block_time.swap(0, Ordering::Relaxed); + let num_get_rooted_transaction = self.num_get_rooted_transaction.swap(0, Ordering::Relaxed); + let num_get_rooted_transaction_status = self + .num_get_rooted_transaction_status + .swap(0, Ordering::Relaxed); + let num_get_rooted_block_with_entries = self + .num_get_rooted_block_with_entries + .swap(0, Ordering::Relaxed); + let num_get_transaction_status = self.num_get_transaction_status.swap(0, Ordering::Relaxed); + + let total_num_queries = num_get_block_height + .saturating_add(num_get_complete_transaction) + .saturating_add(num_get_confirmed_signatures_for_address) + .saturating_add(num_get_confirmed_signatures_for_address2) + .saturating_add(num_get_rooted_block) + .saturating_add(num_get_rooted_block_time) + .saturating_add(num_get_rooted_transaction) + .saturating_add(num_get_rooted_transaction_status) + .saturating_add(num_get_rooted_block_with_entries) + .saturating_add(num_get_transaction_status); + + if total_num_queries > 0 { + datapoint_info!( + "blockstore-rpc-api", + ("num_get_block_height", num_get_block_height as i64, i64), + ( + "num_get_complete_transaction", + num_get_complete_transaction as i64, + i64 + ), + ( + "num_get_confirmed_signatures_for_address", + num_get_confirmed_signatures_for_address as i64, + i64 + ), + ( + "num_get_confirmed_signatures_for_address2", + num_get_confirmed_signatures_for_address2 as i64, + i64 + ), + ("num_get_rooted_block", num_get_rooted_block as i64, i64), + ( + "num_get_rooted_block_time", + num_get_rooted_block_time as i64, + i64 + ), + ( + "num_get_rooted_transaction", + num_get_rooted_transaction as i64, + i64 + ), + ( + "num_get_rooted_transaction_status", + num_get_rooted_transaction_status as i64, + i64 + ), + ( + "num_get_rooted_block_with_entries", + num_get_rooted_block_with_entries as i64, + i64 + ), + ( + "num_get_transaction_status", + num_get_transaction_status as i64, + i64 + ), + ); + } + } +} + /// A metrics struct that exposes RocksDB's column family properties. /// /// Here we only expose a subset of all the internal properties which are From 4bb1aa277c617dd41f862cf9fb06dee17a8691d4 Mon Sep 17 00:00:00 2001 From: Steven Czabaniuk Date: Mon, 13 Nov 2023 12:39:49 -0600 Subject: [PATCH 2/3] Plumb common RPC API metrics struct instead of individual datapoints --- ledger/src/blockstore.rs | 79 +++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 3010a65be7f90c..ae27fa56c3db30 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -10,6 +10,7 @@ use { IteratorMode, LedgerColumn, Result, WriteBatch, }, blockstore_meta::*, + blockstore_metrics::BlockstoreRpcApiMetrics, blockstore_options::{ AccessType, BlockstoreOptions, LedgerColumnOptions, BLOCKSTORE_DIRECTORY_ROCKS_FIFO, BLOCKSTORE_DIRECTORY_ROCKS_LEVEL, @@ -222,6 +223,7 @@ pub struct Blockstore { pub shred_timing_point_sender: Option, pub lowest_cleanup_slot: RwLock, pub slots_stats: SlotsStats, + rpc_api_metrics: BlockstoreRpcApiMetrics, } pub struct IndexMetaWorkingSetEntry { @@ -362,6 +364,7 @@ impl Blockstore { max_root, lowest_cleanup_slot: RwLock::::default(), slots_stats: SlotsStats::default(), + rpc_api_metrics: BlockstoreRpcApiMetrics::default(), }; blockstore.cleanup_old_entries()?; blockstore.update_highest_primary_index_slot()?; @@ -717,6 +720,11 @@ impl Blockstore { self.merkle_root_meta_cf.submit_rocksdb_cf_metrics(); } + /// Report the accumulated RPC API metrics + pub(crate) fn report_rpc_api_metrics(&self) { + self.rpc_api_metrics.report(); + } + fn try_shred_recovery( &self, erasure_metas: &HashMap, @@ -1961,10 +1969,9 @@ impl Blockstore { } pub fn get_rooted_block_time(&self, slot: Slot) -> Result { - datapoint_info!( - "blockstore-rpc-api", - ("method", "get_rooted_block_time", String) - ); + self.rpc_api_metrics + .num_get_rooted_block_time + .fetch_add(1, Ordering::Relaxed); let _lock = self.check_lowest_cleanup_slot(slot)?; if self.is_root(slot) { @@ -1981,8 +1988,11 @@ impl Blockstore { } pub fn get_block_height(&self, slot: Slot) -> Result> { - datapoint_info!("blockstore-rpc-api", ("method", "get_block_height", String)); + self.rpc_api_metrics + .num_get_block_height + .fetch_add(1, Ordering::Relaxed); let _lock = self.check_lowest_cleanup_slot(slot)?; + self.block_height_cf.get(slot) } @@ -2010,7 +2020,9 @@ impl Blockstore { slot: Slot, require_previous_blockhash: bool, ) -> Result { - datapoint_info!("blockstore-rpc-api", ("method", "get_rooted_block", String)); + self.rpc_api_metrics + .num_get_rooted_block + .fetch_add(1, Ordering::Relaxed); let _lock = self.check_lowest_cleanup_slot(slot)?; if self.is_root(slot) { @@ -2033,10 +2045,9 @@ impl Blockstore { slot: Slot, require_previous_blockhash: bool, ) -> Result { - datapoint_info!( - "blockstore-rpc-api", - ("method", "get_rooted_block_with_entries", String) - ); + self.rpc_api_metrics + .num_get_rooted_block_with_entries + .fetch_add(1, Ordering::Relaxed); let _lock = self.check_lowest_cleanup_slot(slot)?; if self.is_root(slot) { @@ -2441,10 +2452,10 @@ impl Blockstore { &self, signature: Signature, ) -> Result> { - datapoint_info!( - "blockstore-rpc-api", - ("method", "get_rooted_transaction_status", String) - ); + self.rpc_api_metrics + .num_get_rooted_transaction_status + .fetch_add(1, Ordering::Relaxed); + self.get_transaction_status(signature, &HashSet::default()) } @@ -2454,10 +2465,10 @@ impl Blockstore { signature: Signature, confirmed_unrooted_slots: &HashSet, ) -> Result> { - datapoint_info!( - "blockstore-rpc-api", - ("method", "get_transaction_status", String) - ); + self.rpc_api_metrics + .num_get_transaction_status + .fetch_add(1, Ordering::Relaxed); + self.get_transaction_status_with_counter(signature, confirmed_unrooted_slots) .map(|(status, _)| status) } @@ -2467,10 +2478,10 @@ impl Blockstore { &self, signature: Signature, ) -> Result> { - datapoint_info!( - "blockstore-rpc-api", - ("method", "get_rooted_transaction", String) - ); + self.rpc_api_metrics + .num_get_rooted_transaction + .fetch_add(1, Ordering::Relaxed); + self.get_transaction_with_status(signature, &HashSet::default()) } @@ -2480,10 +2491,10 @@ impl Blockstore { signature: Signature, highest_confirmed_slot: Slot, ) -> Result> { - datapoint_info!( - "blockstore-rpc-api", - ("method", "get_complete_transaction", String) - ); + self.rpc_api_metrics + .num_get_complete_transaction + .fetch_add(1, Ordering::Relaxed); + let max_root = self.max_root(); let confirmed_unrooted_slots: HashSet<_> = AncestorIterator::new_inclusive(highest_confirmed_slot, self) @@ -2593,10 +2604,10 @@ impl Blockstore { start_slot: Slot, end_slot: Slot, ) -> Result> { - datapoint_info!( - "blockstore-rpc-api", - ("method", "get_confirmed_signatures_for_address", String) - ); + self.rpc_api_metrics + .num_get_confirmed_signatures_for_address + .fetch_add(1, Ordering::Relaxed); + self.find_address_signatures(pubkey, start_slot, end_slot) .map(|signatures| signatures.iter().map(|(_, signature)| *signature).collect()) } @@ -2631,10 +2642,10 @@ impl Blockstore { until: Option, limit: usize, ) -> Result { - datapoint_info!( - "blockstore-rpc-api", - ("method", "get_confirmed_signatures_for_address2", String) - ); + self.rpc_api_metrics + .num_get_confirmed_signatures_for_address2 + .fetch_add(1, Ordering::Relaxed); + let max_root = self.max_root(); let confirmed_unrooted_slots: HashSet<_> = AncestorIterator::new_inclusive(highest_slot, self) From 55b823b82d74f649ac847572e79b81180224c16d Mon Sep 17 00:00:00 2001 From: Steven Czabaniuk Date: Mon, 13 Nov 2023 12:40:43 -0600 Subject: [PATCH 3/3] Make RPC API metrics get submitted every 10 seconds Note that the datapoint will not be sent if all of the fields are 0 --- ledger/src/blockstore_metric_report_service.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/ledger/src/blockstore_metric_report_service.rs b/ledger/src/blockstore_metric_report_service.rs index 393442a3e25aca..b08dab1ad38dc5 100644 --- a/ledger/src/blockstore_metric_report_service.rs +++ b/ledger/src/blockstore_metric_report_service.rs @@ -34,6 +34,7 @@ impl BlockstoreMetricReportService { BLOCKSTORE_METRICS_REPORT_PERIOD_MILLIS, )); blockstore.submit_rocksdb_cf_metrics_for_all_cfs(); + blockstore.report_rpc_api_metrics(); }) .unwrap(); Self { t_cf_metric }