Merge pull request #2210 from maqi/pick_PR2088_stable

chore: pick the missing work of PR2088 (auditor optimization)
maidsafe · Oct 9, 2024 · 4ffd314 · 4ffd314
2 parents 0d52e8b + 89760a2
commit 4ffd314
Show file tree

Hide file tree

Showing 12 changed files with 161 additions and 105 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 *When editing this file, please respect a line length of 100.*
 
+## 2024-10-08
+
+### Network
+
+#### Changed
+
+- Optimize auditor tracking by not to re-attempt fetched spend.
+- Optimize auditor tracking function by using DashMap and stream.
+
 ## 2024-10-07
 
 ### Network

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/autonomi/Cargo.toml b/autonomi/Cargo.toml
@@ -28,7 +28,7 @@ rand = "0.8.5"
 rmp-serde = "1.1.1"
 self_encryption = "~0.30.0"
 serde = { version = "1.0.133", features = ["derive", "rc"] }
-sn_client = { path = "../sn_client", version = "0.110.3" }
+sn_client = { path = "../sn_client", version = "0.110.4" }
 sn_protocol = { version = "0.17.11", path = "../sn_protocol" }
 sn_registers = { path = "../sn_registers", version = "0.3.21" }
 sn_transfers = { path = "../sn_transfers", version = "0.19.3" }

diff --git a/release-cycle-info b/release-cycle-info
@@ -15,4 +15,4 @@
 release-year: 2024
 release-month: 10
 release-cycle: 2
-release-cycle-counter: 2
+release-cycle-counter: 3
diff --git a/sn_auditor/Cargo.toml b/sn_auditor/Cargo.toml
@@ -2,7 +2,7 @@
 authors = ["MaidSafe Developers <dev@maidsafe.net>"]
 description = "Safe Network Auditor"
 name = "sn_auditor"
-version = "0.3.4"
+version = "0.3.5"
 edition = "2021"
 homepage = "https://maidsafe.net"
 repository = "https://github.com/maidsafe/safe_network"
@@ -31,7 +31,7 @@ graphviz-rust = { version = "0.9.0", optional = true }
 lazy_static = "1.4.0"
 serde = { version = "1.0.133", features = ["derive", "rc"] }
 serde_json = "1.0.108"
-sn_client = { path = "../sn_client", version = "0.110.3" }
+sn_client = { path = "../sn_client", version = "0.110.4" }
 sn_logging = { path = "../sn_logging", version = "0.2.36" }
 sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.5.3" }
 tiny_http = { version = "0.12", features = ["ssl-rustls"] }

diff --git a/sn_auditor/src/dag_db.rs b/sn_auditor/src/dag_db.rs
@@ -303,6 +303,7 @@ impl SpendDagDb {
         };
 
         let mut addrs_to_get = BTreeMap::new();
+        let mut addrs_fetched = BTreeSet::new();
 
         loop {
             // get expired utxos for re-attempt fetch
@@ -350,16 +351,28 @@ impl SpendDagDb {
                     )
                 }));
             } else if let Some(sender) = spend_processing.clone() {
-                let (reattempt_addrs, fetched_addrs) = client
-                    .crawl_to_next_utxos(&mut addrs_to_get, sender.clone(), *UTXO_REATTEMPT_SECONDS)
+                let (reattempt_addrs, fetched_addrs, addrs_for_further_track) = client
+                    .crawl_to_next_utxos(
+                        addrs_to_get.clone(),
+                        sender.clone(),
+                        *UTXO_REATTEMPT_SECONDS,
+                    )
                     .await;
+
+                addrs_to_get.clear();
                 let mut utxo_addresses = self.utxo_addresses.write().await;
-                for addr in fetched_addrs.iter() {
-                    let _ = utxo_addresses.remove(addr);
+                for addr in fetched_addrs {
+                    let _ = utxo_addresses.remove(&addr);
+                    let _ = addrs_fetched.insert(addr);
                 }
                 for (addr, tuple) in reattempt_addrs {
                     let _ = utxo_addresses.insert(addr, tuple);
                 }
+                for (addr, amount) in addrs_for_further_track {
+                    if !addrs_fetched.contains(&addr) {
+                        let _ = addrs_to_get.entry(addr).or_insert((0, amount));
+                    }
+                }
             } else {
                 panic!("There is no point in running the auditor if we are not collecting the DAG or collecting data through crawling. Please enable the `dag-collection` feature or provide beta program related arguments.");
             };

diff --git a/sn_cli/Cargo.toml b/sn_cli/Cargo.toml
@@ -55,7 +55,7 @@ rmp-serde = "1.1.1"
 rpassword = "7.3.1"
 serde = { version = "1.0.133", features = ["derive"] }
 sn_build_info = { path = "../sn_build_info", version = "0.1.15" }
-sn_client = { path = "../sn_client", version = "0.110.3" }
+sn_client = { path = "../sn_client", version = "0.110.4" }
 sn_logging = { path = "../sn_logging", version = "0.2.36" }
 sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.5.3" }
 sn_protocol = { path = "../sn_protocol", version = "0.17.11" }
@@ -80,7 +80,7 @@ eyre = "0.6.8"
 criterion = "0.5.1"
 tempfile = "3.6.0"
 rand = { version = "~0.8.5", features = ["small_rng"] }
-sn_client = { path = "../sn_client", version = "0.110.3", features = [
+sn_client = { path = "../sn_client", version = "0.110.4", features = [
     "test-utils",
 ] }
 

diff --git a/sn_client/Cargo.toml b/sn_client/Cargo.toml
@@ -8,7 +8,7 @@ license = "GPL-3.0"
 name = "sn_client"
 readme = "README.md"
 repository = "https://github.com/maidsafe/safe_network"
-version = "0.110.3"
+version = "0.110.4"
 
 [features]
 default = []
@@ -38,6 +38,7 @@ bls = { package = "blsttc", version = "8.0.1" }
 bytes = { version = "1.0.1", features = ["serde"] }
 crdts = "7.3.2"
 custom_debug = "~0.6.1"
+dashmap = "~6.1.0"
 futures = "~0.3.13"
 hex = "~0.4.3"
 itertools = "~0.12.1"