Log tx script cache stats (wip)

michaelsutton · Sep 24, 2023 · c29680f · c29680f
1 parent abd3143
commit c29680f
Show file tree

Hide file tree

Showing 16 changed files with 201 additions and 39 deletions.
diff --git a/consensus/src/consensus/factory.rs b/consensus/src/consensus/factory.rs
@@ -11,6 +11,7 @@ use kaspa_database::{
     registry::DatabaseStorePrefixes,
 };
 
+use kaspa_txscript::caches::TxScriptCacheCounters;
 use parking_lot::RwLock;
 use rocksdb::WriteBatch;
 use serde::{Deserialize, Serialize};
@@ -153,6 +154,7 @@ pub struct Factory {
     db_parallelism: usize,
     notification_root: Arc<ConsensusNotificationRoot>,
     counters: Arc<ProcessingCounters>,
+    tx_script_cache_counters: Arc<TxScriptCacheCounters>,
 }
 
 impl Factory {
@@ -163,6 +165,7 @@ impl Factory {
         db_parallelism: usize,
         notification_root: Arc<ConsensusNotificationRoot>,
         counters: Arc<ProcessingCounters>,
+        tx_script_cache_counters: Arc<TxScriptCacheCounters>,
     ) -> Self {
         let mut config = config.clone();
         #[cfg(feature = "devnet-prealloc")]
@@ -175,6 +178,7 @@ impl Factory {
             db_parallelism,
             notification_root,
             counters,
+            tx_script_cache_counters,
         }
     }
 }
@@ -208,6 +212,7 @@ impl ConsensusFactory for Factory {
             session_lock.clone(),
             self.notification_root.clone(),
             self.counters.clone(),
+            self.tx_script_cache_counters.clone(),
             entry.creation_timestamp,
         ));
 
@@ -236,6 +241,7 @@ impl ConsensusFactory for Factory {
             session_lock.clone(),
             self.notification_root.clone(),
             self.counters.clone(),
+            self.tx_script_cache_counters.clone(),
             entry.creation_timestamp,
         ));
 

diff --git a/consensus/src/consensus/mod.rs b/consensus/src/consensus/mod.rs
@@ -70,6 +70,7 @@ use kaspa_consensusmanager::{SessionLock, SessionReadGuard};
 use kaspa_database::prelude::StoreResultExtensions;
 use kaspa_hashes::Hash;
 use kaspa_muhash::MuHash;
+use kaspa_txscript::caches::TxScriptCacheCounters;
 
 use std::thread::{self, JoinHandle};
 use std::{
@@ -132,6 +133,7 @@ impl Consensus {
         pruning_lock: SessionLock,
         notification_root: Arc<ConsensusNotificationRoot>,
         counters: Arc<ProcessingCounters>,
+        tx_script_cache_counters: Arc<TxScriptCacheCounters>,
         creation_timestamp: u64,
     ) -> Self {
         let params = &config.params;
@@ -147,7 +149,7 @@ impl Consensus {
         // Services and managers
         //
 
-        let services = ConsensusServices::new(db.clone(), storage.clone(), config.clone());
+        let services = ConsensusServices::new(db.clone(), storage.clone(), config.clone(), tx_script_cache_counters);
 
         //
         // Processor channels

diff --git a/consensus/src/consensus/services.rs b/consensus/src/consensus/services.rs
@@ -18,6 +18,7 @@ use crate::{
 };
 
 use itertools::Itertools;
+use kaspa_txscript::caches::TxScriptCacheCounters;
 use std::sync::Arc;
 
 pub type DbGhostdagManager =
@@ -65,7 +66,12 @@ pub struct ConsensusServices {
 }
 
 impl ConsensusServices {
-    pub fn new(db: Arc<DB>, storage: Arc<ConsensusStorage>, config: Arc<Config>) -> Arc<Self> {
+    pub fn new(
+        db: Arc<DB>,
+        storage: Arc<ConsensusStorage>,
+        config: Arc<Config>,
+        tx_script_cache_counters: Arc<TxScriptCacheCounters>,
+    ) -> Arc<Self> {
         let params = &config.params;
 
         let statuses_service = MTStatusesService::new(storage.statuses_store.clone());
@@ -144,6 +150,7 @@ impl ConsensusServices {
             params.ghostdag_k,
             params.coinbase_payload_script_public_key_max_len,
             params.coinbase_maturity,
+            tx_script_cache_counters,
         );
 
         let pruning_point_manager = PruningPointManager::new(

diff --git a/consensus/src/consensus/test_consensus.rs b/consensus/src/consensus/test_consensus.rs
@@ -49,8 +49,17 @@ impl TestConsensus {
     /// Creates a test consensus instance based on `config` with the provided `db` and `notification_sender`
     pub fn with_db(db: Arc<DB>, config: &Config, notification_sender: Sender<Notification>) -> Self {
         let notification_root = Arc::new(ConsensusNotificationRoot::new(notification_sender));
-        let counters = Arc::new(ProcessingCounters::default());
-        let consensus = Arc::new(Consensus::new(db, Arc::new(config.clone()), Default::default(), notification_root, counters, 0));
+        let counters = Default::default();
+        let tx_script_cache_counters = Default::default();
+        let consensus = Arc::new(Consensus::new(
+            db,
+            Arc::new(config.clone()),
+            Default::default(),
+            notification_root,
+            counters,
+            tx_script_cache_counters,
+            0,
+        ));
         let block_builder = TestBlockBuilder::new(consensus.virtual_processor.clone());
 
         Self { params: config.params.clone(), consensus, block_builder, db_lifetime: Default::default() }
@@ -60,8 +69,17 @@ impl TestConsensus {
     pub fn with_notifier(config: &Config, notification_sender: Sender<Notification>) -> Self {
         let (db_lifetime, db) = create_temp_db!(ConnBuilder::default());
         let notification_root = Arc::new(ConsensusNotificationRoot::new(notification_sender));
-        let counters = Arc::new(ProcessingCounters::default());
-        let consensus = Arc::new(Consensus::new(db, Arc::new(config.clone()), Default::default(), notification_root, counters, 0));
+        let counters = Default::default();
+        let tx_script_cache_counters = Default::default();
+        let consensus = Arc::new(Consensus::new(
+            db,
+            Arc::new(config.clone()),
+            Default::default(),
+            notification_root,
+            counters,
+            tx_script_cache_counters,
+            0,
+        ));
         let block_builder = TestBlockBuilder::new(consensus.virtual_processor.clone());
 
         Self { consensus, block_builder, params: config.params.clone(), db_lifetime }
@@ -72,8 +90,17 @@ impl TestConsensus {
         let (db_lifetime, db) = create_temp_db!(ConnBuilder::default());
         let (dummy_notification_sender, _) = async_channel::unbounded();
         let notification_root = Arc::new(ConsensusNotificationRoot::new(dummy_notification_sender));
-        let counters = Arc::new(ProcessingCounters::default());
-        let consensus = Arc::new(Consensus::new(db, Arc::new(config.clone()), Default::default(), notification_root, counters, 0));
+        let counters = Default::default();
+        let tx_script_cache_counters = Default::default();
+        let consensus = Arc::new(Consensus::new(
+            db,
+            Arc::new(config.clone()),
+            Default::default(),
+            notification_root,
+            counters,
+            tx_script_cache_counters,
+            0,
+        ));
         let block_builder = TestBlockBuilder::new(consensus.virtual_processor.clone());
 
         Self { consensus, block_builder, params: config.params.clone(), db_lifetime }

diff --git a/consensus/src/processes/transaction_validator/mod.rs b/consensus/src/processes/transaction_validator/mod.rs
@@ -2,9 +2,14 @@ pub mod errors;
 pub mod transaction_validator_populated;
 mod tx_validation_in_isolation;
 pub mod tx_validation_not_utxo_related;
+use std::sync::Arc;
+
 use crate::model::stores::ghostdag;
 
-use kaspa_txscript::{caches::Cache, SigCacheKey};
+use kaspa_txscript::{
+    caches::{Cache, TxScriptCacheCounters},
+    SigCacheKey,
+};
 pub use tx_validation_in_isolation::*;
 
 #[derive(Clone)]
@@ -28,6 +33,7 @@ impl TransactionValidator {
         ghostdag_k: ghostdag::KType,
         coinbase_payload_script_public_key_max_len: u8,
         coinbase_maturity: u64,
+        counters: Arc<TxScriptCacheCounters>,
     ) -> Self {
         Self {
             max_tx_inputs,
@@ -37,7 +43,7 @@ impl TransactionValidator {
             ghostdag_k,
             coinbase_payload_script_public_key_max_len,
             coinbase_maturity,
-            sig_cache: Cache::new(10_000),
+            sig_cache: Cache::new(10_000, counters),
         }
     }
 }
diff --git a/consensus/src/processes/transaction_validator/transaction_validator_populated.rs b/consensus/src/processes/transaction_validator/transaction_validator_populated.rs
@@ -142,6 +142,7 @@ mod tests {
             params.ghostdag_k,
             params.coinbase_payload_script_public_key_max_len,
             params.coinbase_maturity,
+            Default::default(),
         );
 
         let prev_tx_id = TransactionId::from_str("746915c8dfc5e1550eacbe1d87625a105750cf1a65aaddd1baa60f8bcf7e953c").unwrap();
@@ -202,6 +203,7 @@ mod tests {
             params.ghostdag_k,
             params.coinbase_payload_script_public_key_max_len,
             params.coinbase_maturity,
+            Default::default(),
         );
 
         // Taken from: 3f582463d73c77d93f278b7bf649bd890e75fe9bb8a1edd7a6854df1a2a2bfc1
@@ -263,6 +265,7 @@ mod tests {
             params.ghostdag_k,
             params.coinbase_payload_script_public_key_max_len,
             params.coinbase_maturity,
+            Default::default(),
         );
 
         // Taken from: d839d29b549469d0f9a23e51febe68d4084967a6a477868b511a5a8d88c5ae06
@@ -324,6 +327,7 @@ mod tests {
             params.ghostdag_k,
             params.coinbase_payload_script_public_key_max_len,
             params.coinbase_maturity,
+            Default::default(),
         );
 
         // Taken from: d839d29b549469d0f9a23e51febe68d4084967a6a477868b511a5a8d88c5ae06
@@ -386,6 +390,7 @@ mod tests {
             params.ghostdag_k,
             params.coinbase_payload_script_public_key_max_len,
             params.coinbase_maturity,
+            Default::default(),
         );
 
         // Taken from: d839d29b549469d0f9a23e51febe68d4084967a6a477868b511a5a8d88c5ae06
@@ -448,6 +453,7 @@ mod tests {
             params.ghostdag_k,
             params.coinbase_payload_script_public_key_max_len,
             params.coinbase_maturity,
+            Default::default(),
         );
 
         // Taken from: d839d29b549469d0f9a23e51febe68d4084967a6a477868b511a5a8d88c5ae06
@@ -510,6 +516,7 @@ mod tests {
             params.ghostdag_k,
             params.coinbase_payload_script_public_key_max_len,
             params.coinbase_maturity,
+            Default::default(),
         );
 
         let prev_tx_id = TransactionId::from_str("1111111111111111111111111111111111111111111111111111111111111111").unwrap();
@@ -563,6 +570,7 @@ mod tests {
             params.ghostdag_k,
             params.coinbase_payload_script_public_key_max_len,
             params.coinbase_maturity,
+            Default::default(),
         );
 
         let secp = Secp256k1::new();

diff --git a/consensus/src/processes/transaction_validator/tx_validation_in_isolation.rs b/consensus/src/processes/transaction_validator/tx_validation_in_isolation.rs
@@ -173,6 +173,7 @@ mod tests {
             params.ghostdag_k,
             params.coinbase_payload_script_public_key_max_len,
             params.coinbase_maturity,
+            Default::default(),
         );
 
         let valid_cb = Transaction::new(

diff --git a/crypto/txscript/src/caches.rs b/crypto/txscript/src/caches.rs
@@ -1,22 +1,37 @@
 use indexmap::IndexMap;
 use parking_lot::RwLock;
 use rand::Rng;
-use std::{collections::hash_map::RandomState, hash::BuildHasher, sync::Arc};
+use std::{
+    collections::hash_map::RandomState,
+    hash::BuildHasher,
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        Arc,
+    },
+};
 
 #[derive(Clone)]
 pub struct Cache<TKey: Clone + std::hash::Hash + Eq + Send + Sync, TData: Clone + Send + Sync, S = RandomState> {
     // We use IndexMap and not HashMap, because it makes it cheaper to remove a random element when the cache is full.
     map: Arc<RwLock<IndexMap<TKey, TData, S>>>,
     size: usize,
+    counters: Arc<TxScriptCacheCounters>,
 }
 
 impl<TKey: Clone + std::hash::Hash + Eq + Send + Sync, TData: Clone + Send + Sync, S: BuildHasher + Default> Cache<TKey, TData, S> {
-    pub fn new(size: u64) -> Self {
-        Self { map: Arc::new(RwLock::new(IndexMap::with_capacity_and_hasher(size as usize, S::default()))), size: size as usize }
+    pub fn new(size: u64, counters: Arc<TxScriptCacheCounters>) -> Self {
+        Self {
+            map: Arc::new(RwLock::new(IndexMap::with_capacity_and_hasher(size as usize, S::default()))),
+            size: size as usize,
+            counters,
+        }
     }
 
     pub(crate) fn get(&self, key: &TKey) -> Option<TData> {
-        self.map.read().get(key).cloned()
+        self.map.read().get(key).cloned().map(|data| {
+            self.counters.get_counts.fetch_add(1, Ordering::Relaxed);
+            data
+        })
     }
 
     pub(crate) fn insert(&self, key: TKey, data: TData) {
@@ -28,5 +43,48 @@ impl<TKey: Clone + std::hash::Hash + Eq + Send + Sync, TData: Clone + Send + Syn
             write_guard.swap_remove_index(rand::thread_rng().gen_range(0..self.size));
         }
         write_guard.insert(key, data);
+        self.counters.insert_counts.fetch_add(1, Ordering::Relaxed);
+    }
+}
+
+#[derive(Default)]
+pub struct TxScriptCacheCounters {
+    pub insert_counts: AtomicU64,
+    pub get_counts: AtomicU64,
+}
+
+impl TxScriptCacheCounters {
+    pub fn snapshot(&self) -> TxScriptCacheCountersSnapshot {
+        TxScriptCacheCountersSnapshot {
+            insert_counts: self.insert_counts.load(Ordering::Relaxed),
+            get_counts: self.get_counts.load(Ordering::Relaxed),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct TxScriptCacheCountersSnapshot {
+    pub insert_counts: u64,
+    pub get_counts: u64,
+}
+
+impl TxScriptCacheCountersSnapshot {
+    pub fn hit_ratio(&self) -> f64 {
+        if self.insert_counts > 0 {
+            self.get_counts as f64 / self.insert_counts as f64
+        } else {
+            0f64
+        }
+    }
+}
+
+impl core::ops::Sub for &TxScriptCacheCountersSnapshot {
+    type Output = TxScriptCacheCountersSnapshot;
+
+    fn sub(self, rhs: Self) -> Self::Output {
+        Self::Output {
+            insert_counts: self.insert_counts.checked_sub(rhs.insert_counts).unwrap_or_default(),
+            get_counts: self.get_counts.checked_sub(rhs.get_counts).unwrap_or_default(),
+        }
     }
 }
diff --git a/crypto/txscript/src/lib.rs b/crypto/txscript/src/lib.rs
@@ -529,7 +529,7 @@ mod tests {
     }
 
     fn run_test_script_cases(test_cases: Vec<ScriptTestCase>) {
-        let sig_cache = Cache::new(10_000);
+        let sig_cache = Cache::new(10_000, Default::default());
         let mut reused_values = SigHashReusedValues::new();
 
         for test in test_cases {