Merge remote-tracking branch 'maidsafe/main' into feat-upgradable-sma…

…rt-contracts-and-updated-quotation-flow-rebased

# Conflicts:
#	Cargo.lock
#	ant-evm/src/data_payments.rs
#	ant-networking/src/cmd.rs
#	ant-networking/src/record_store.rs
#	ant-networking/src/record_store_api.rs

ant-evm/Cargo.toml

@@ -19,7 +19,7 @@ custom_debug = "~0.6.1"
 evmlib = { path = "../evmlib", version = "0.1.4" }
 hex = "~0.4.3"
 lazy_static = "~1.4.0"
-libp2p = { git = "https://github.com/maqi/rust-libp2p.git", branch = "master", features = ["identify", "kad"] }
+libp2p = { version = "0.54.1", features = ["identify", "kad"] }
 rand = { version = "~0.8.5", features = ["small_rng"] }
 ring = "0.17.8"
 rmp-serde = "1.1.1"

ant-evm/src/lib.rs

@@ -12,6 +12,7 @@ extern crate tracing;
 pub use evmlib::common::Address as RewardsAddress;
 pub use evmlib::common::Address as EvmAddress;
 pub use evmlib::common::QuotePayment;
+pub use evmlib::common::U256;
 pub use evmlib::common::{QuoteHash, TxHash};
 pub use evmlib::contract::payment_vault;
 pub use evmlib::cryptography;

ant-networking/Cargo.toml

@@ -39,7 +39,7 @@ hyper = { version = "0.14", features = [
 ], optional = true }
 itertools = "~0.12.1"
 lazy_static = "~1.4.0"
-libp2p = { git = "https://github.com/maqi/rust-libp2p.git", branch = "master", features = [
+libp2p = { version = "0.54.1", features = [
     "tokio",
     "dns",
     "kad",
@@ -95,7 +95,7 @@ crate-type = ["cdylib", "rlib"]
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 getrandom = { version = "0.2.12", features = ["js"] }
-libp2p = { git = "https://github.com/maqi/rust-libp2p.git", branch = "master", features = [
+libp2p = { version = "0.54.1", features = [
     "tokio",
     "dns",
     "kad",

ant-networking/src/cmd.rs

@@ -13,8 +13,9 @@ use crate::{
     log_markers::Marker,
     multiaddr_pop_p2p, GetRecordCfg, GetRecordError, MsgResponder, NetworkEvent, CLOSE_GROUP_SIZE,
 };
-use ant_evm::{PaymentQuote, QuotingMetrics};
+use ant_evm::{PaymentQuote, QuotingMetrics, U256};
 use ant_protocol::{
+    convert_distance_to_u256,
     messages::{Cmd, Request, Response},
     storage::{RecordHeader, RecordKind, RecordType},
     NetworkAddress, PrettyPrintRecordKey,
@@ -1138,11 +1139,12 @@ impl SwarmDriver {
 }
 
 /// Returns the nodes that within the defined distance.
-fn get_peers_in_range(peers: &[PeerId], address: &NetworkAddress, range: Distance) -> Vec<PeerId> {
+fn get_peers_in_range(peers: &[PeerId], address: &NetworkAddress, range: U256) -> Vec<PeerId> {
     peers
         .iter()
         .filter_map(|peer_id| {
-            let distance = address.distance(&NetworkAddress::from_peer(*peer_id));
+            let distance =
+                convert_distance_to_u256(&address.distance(&NetworkAddress::from_peer(*peer_id)));
             if distance <= range {
                 Some(*peer_id)
             } else {

ant-networking/src/driver.rs

@@ -30,8 +30,9 @@ use crate::{
     metrics::service::run_metrics_server, metrics::NetworkMetricsRecorder, MetricsRegistries,
 };
 use ant_bootstrap::BootstrapCacheStore;
-use ant_evm::PaymentQuote;
+use ant_evm::{PaymentQuote, U256};
 use ant_protocol::{
+    convert_distance_to_u256,
     messages::{ChunkProof, Nonce, Request, Response},
     storage::{try_deserialize_record, RetryStrategy},
     version::{
@@ -48,7 +49,7 @@ use libp2p::mdns;
 use libp2p::{core::muxing::StreamMuxerBox, relay};
 use libp2p::{
     identity::Keypair,
-    kad::{self, KBucketDistance as Distance, QueryId, Quorum, Record, RecordKey, K_VALUE, U256},
+    kad::{self, QueryId, Quorum, Record, RecordKey, K_VALUE},
     multiaddr::Protocol,
     request_response::{self, Config as RequestResponseConfig, OutboundRequestId, ProtocolSupport},
     swarm::{
@@ -973,10 +974,9 @@ impl SwarmDriver {
                         // The network density (average distance among nodes) can be estimated as:
                         //     network_density = entire_U256_space / estimated_network_size
                         let density = U256::MAX / U256::from(estimated_network_size);
-                        let estimated_distance = density * U256::from(CLOSE_GROUP_SIZE);
-                        let density_distance = Distance(estimated_distance);
+                        let density_distance = density * U256::from(CLOSE_GROUP_SIZE);
 
-                        // Use distanct to close peer to avoid the situation that
+                        // Use distance to close peer to avoid the situation that
                         // the estimated density_distance is too narrow.
                         let closest_k_peers = self.get_closest_k_value_local_peers();
                         if closest_k_peers.len() <= CLOSE_GROUP_SIZE + 2 {
@@ -986,9 +986,12 @@ impl SwarmDriver {
                         // Note: self is included
                         let self_addr = NetworkAddress::from_peer(self.self_peer_id);
                         let close_peers_distance = self_addr.distance(&NetworkAddress::from_peer(closest_k_peers[CLOSE_GROUP_SIZE + 1]));
+                        let close_peers_u256 = convert_distance_to_u256(&close_peers_distance);
 
-                        let distance = std::cmp::max(density_distance, close_peers_distance);
+                        let distance = std::cmp::max(density_distance, close_peers_u256);
 
+                        // The sampling approach has severe impact to the node side performance
+                        // Hence suggested to be only used by client side.
                         // let distance = if let Some(distance) = self.network_density_samples.get_median() {
                         //     distance
                         // } else {
@@ -1002,10 +1005,9 @@ impl SwarmDriver {
                         //     // Note: self is included
                         //     let self_addr = NetworkAddress::from_peer(self.self_peer_id);
                         //     self_addr.distance(&NetworkAddress::from_peer(closest_k_peers[CLOSE_GROUP_SIZE]))
-
                         // };
 
-                        info!("Set responsible range to {distance:?}({:?})", distance.ilog2());
+                        info!("Set responsible range to {distance:?}({:?})", distance.log2());
 
                         // set any new distance to farthest record in the store
                         self.swarm.behaviour_mut().kademlia.store_mut().set_distance_range(distance);

ant-networking/src/record_store.rs

@@ -16,8 +16,9 @@ use aes_gcm_siv::{
     aead::{Aead, KeyInit},
     Aes256GcmSiv, Key as AesKey, Nonce,
 };
-use ant_evm::QuotingMetrics;
+use ant_evm::{QuotingMetrics, U256};
 use ant_protocol::{
+    convert_distance_to_u256,
     storage::{RecordHeader, RecordKind, RecordType},
     NetworkAddress, PrettyPrintRecordKey,
 };
@@ -139,7 +140,7 @@ pub struct NodeRecordStore {
     /// Main records store remains unchanged for compatibility
     records: HashMap<Key, (NetworkAddress, RecordType)>,
     /// Additional index organizing records by distance
-    records_by_distance: BTreeMap<Distance, Key>,
+    records_by_distance: BTreeMap<U256, Key>,
     /// FIFO simple cache of records to reduce read times
     records_cache: RecordCache,
     /// Send network events to the node layer.
@@ -149,7 +150,7 @@ pub struct NodeRecordStore {
     /// ilog2 distance range of responsible records
     /// AKA: how many buckets of data do we consider "close"
     /// None means accept all records.
-    responsible_distance_range: Option<Distance>,
+    responsible_distance_range: Option<U256>,
     #[cfg(feature = "open-metrics")]
     /// Used to report the number of records held by the store to the metrics server.
     record_count_metric: Option<Gauge>,
@@ -368,9 +369,9 @@ impl NodeRecordStore {
         let local_address = NetworkAddress::from_peer(local_id);
 
         // Initialize records_by_distance
-        let mut records_by_distance: BTreeMap<Distance, Key> = BTreeMap::new();
+        let mut records_by_distance: BTreeMap<U256, Key> = BTreeMap::new();
         for (key, (addr, _record_type)) in records.iter() {
-            let distance = local_address.distance(addr);
+            let distance = convert_distance_to_u256(&local_address.distance(addr));
             let _ = records_by_distance.insert(distance, key.clone());
         }
 
@@ -407,7 +408,7 @@ impl NodeRecordStore {
     }
 
     /// Returns the current distance ilog2 (aka bucket) range of CLOSE_GROUP nodes.
-    pub fn get_responsible_distance_range(&self) -> Option<Distance> {
+    pub fn get_responsible_distance_range(&self) -> Option<U256> {
         self.responsible_distance_range
     }
 
@@ -609,13 +610,14 @@ impl NodeRecordStore {
     pub(crate) fn mark_as_stored(&mut self, key: Key, record_type: RecordType) {
         let addr = NetworkAddress::from_record_key(&key);
         let distance = self.local_address.distance(&addr);
+        let distance_u256 = convert_distance_to_u256(&distance);
 
         // Update main records store
         self.records
             .insert(key.clone(), (addr.clone(), record_type));
 
         // Update bucket index
-        let _ = self.records_by_distance.insert(distance, key.clone());
+        let _ = self.records_by_distance.insert(distance_u256, key.clone());
 
         // Update farthest record if needed (unchanged)
         if let Some((_farthest_record, farthest_record_distance)) = self.farthest_record.clone() {
@@ -746,7 +748,7 @@ impl NodeRecordStore {
             let relevant_records = self.get_records_within_distance_range(distance_range);
 
             // The `responsible_range` is the network density
-            quoting_metrics.network_density = Some(distance_range.0.into());
+            quoting_metrics.network_density = Some(distance_range.to_be_bytes());
 
             quoting_metrics.close_records_stored = relevant_records;
         } else {
@@ -769,7 +771,7 @@ impl NodeRecordStore {
     }
 
     /// Calculate how many records are stored within a distance range
-    pub fn get_records_within_distance_range(&self, range: Distance) -> usize {
+    pub fn get_records_within_distance_range(&self, range: U256) -> usize {
         let within_range = self
             .records_by_distance
             .range(..range)
@@ -782,7 +784,7 @@ impl NodeRecordStore {
     }
 
     /// Setup the distance range.
-    pub(crate) fn set_responsible_distance_range(&mut self, responsible_distance: Distance) {
+    pub(crate) fn set_responsible_distance_range(&mut self, responsible_distance: U256) {
         self.responsible_distance_range = Some(responsible_distance);
     }
 }
@@ -878,7 +880,7 @@ impl RecordStore for NodeRecordStore {
     fn remove(&mut self, k: &Key) {
         // Remove from main store
         if let Some((addr, _)) = self.records.remove(k) {
-            let distance = self.local_address.distance(&addr);
+            let distance = convert_distance_to_u256(&self.local_address.distance(&addr));
             let _ = self.records_by_distance.remove(&distance);
         }
 
@@ -1001,6 +1003,9 @@ mod tests {
     use bls::SecretKey;
     use xor_name::XorName;
 
+    use ant_evm::utils::dummy_address;
+    use ant_evm::{PaymentQuote, RewardsAddress};
+    use ant_protocol::convert_distance_to_u256;
     use ant_protocol::storage::{
         try_deserialize_record, try_serialize_record, Chunk, ChunkAddress, Scratchpad,
     };
@@ -1569,7 +1574,7 @@ mod tests {
                 .wrap_err("Could not parse record store key")?,
         );
         // get the distance to this record from our local key
-        let distance = self_address.distance(&halfway_record_address);
+        let distance = convert_distance_to_u256(&self_address.distance(&halfway_record_address));
 
         // must be plus one bucket from the halfway record
         store.set_responsible_distance_range(distance);

ant-networking/src/record_store_api.rs

@@ -8,11 +8,11 @@
 #![allow(clippy::mutable_key_type)] // for the Bytes in NetworkAddress
 
 use crate::record_store::{ClientRecordStore, NodeRecordStore};
-use ant_evm::QuotingMetrics;
+use ant_evm::{QuotingMetrics, U256};
 use ant_protocol::{storage::RecordType, NetworkAddress};
 use libp2p::kad::{
     store::{RecordStore, Result},
-    KBucketDistance as Distance, ProviderRecord, Record, RecordKey,
+    ProviderRecord, Record, RecordKey,
 };
 use std::{borrow::Cow, collections::HashMap};
 
@@ -136,7 +136,7 @@ impl UnifiedRecordStore {
         }
     }
 
-    pub(crate) fn get_farthest_replication_distance(&self) -> Option<Distance> {
+    pub(crate) fn get_farthest_replication_distance(&self) -> Option<U256> {
         match self {
             Self::Client(_store) => {
                 warn!("Calling get_distance_range at Client. This should not happen");
@@ -146,7 +146,7 @@ impl UnifiedRecordStore {
         }
     }
 
-    pub(crate) fn set_distance_range(&mut self, distance: Distance) {
+    pub(crate) fn set_distance_range(&mut self, distance: U256) {
         match self {
             Self::Client(_store) => {
                 warn!("Calling set_distance_range at Client. This should not happen");

ant-networking/src/replication_fetcher.rs

@@ -9,7 +9,10 @@
 
 use crate::target_arch::spawn;
 use crate::{event::NetworkEvent, target_arch::Instant};
-use ant_protocol::{storage::RecordType, NetworkAddress, PrettyPrintRecordKey};
+use ant_evm::U256;
+use ant_protocol::{
+    convert_distance_to_u256, storage::RecordType, NetworkAddress, PrettyPrintRecordKey,
+};
 use libp2p::{
     kad::{KBucketDistance as Distance, RecordKey, K_VALUE},
     PeerId,
@@ -42,7 +45,7 @@ pub(crate) struct ReplicationFetcher {
     on_going_fetches: HashMap<(RecordKey, RecordType), (PeerId, ReplicationTimeout)>,
     event_sender: mpsc::Sender<NetworkEvent>,
     /// Distance range that the incoming key shall be fetched
-    distance_range: Option<Distance>,
+    distance_range: Option<U256>,
     /// Restrict fetch range to closer than this value
     /// used when the node is full, but we still have "close" data coming in
     /// that is _not_ closer than our farthest max record
@@ -63,7 +66,7 @@ impl ReplicationFetcher {
     }
 
     /// Set the distance range.
-    pub(crate) fn set_replication_distance_range(&mut self, distance_range: Distance) {
+    pub(crate) fn set_replication_distance_range(&mut self, distance_range: U256) {
         self.distance_range = Some(distance_range);
     }
 
@@ -136,7 +139,8 @@ impl ReplicationFetcher {
         // Filter out those out_of_range ones among the incoming_keys.
         if let Some(ref distance_range) = self.distance_range {
             new_incoming_keys.retain(|(addr, _record_type)| {
-                let is_in_range = self_address.distance(addr) <= *distance_range;
+                let is_in_range =
+                    convert_distance_to_u256(&self_address.distance(addr)) <= *distance_range;
                 if !is_in_range {
                     out_of_range_keys.push(addr.clone());
                 }
@@ -408,7 +412,7 @@ impl ReplicationFetcher {
 #[cfg(test)]
 mod tests {
     use super::{ReplicationFetcher, FETCH_TIMEOUT, MAX_PARALLEL_FETCH};
-    use ant_protocol::{storage::RecordType, NetworkAddress};
+    use ant_protocol::{convert_distance_to_u256, storage::RecordType, NetworkAddress};
     use eyre::Result;
     use libp2p::{kad::RecordKey, PeerId};
     use std::{collections::HashMap, time::Duration};
@@ -479,7 +483,8 @@ mod tests {
         // Set distance range
         let distance_target = NetworkAddress::from_peer(PeerId::random());
         let distance_range = self_address.distance(&distance_target);
-        replication_fetcher.set_replication_distance_range(distance_range);
+        let distance_256 = convert_distance_to_u256(&distance_range);
+        replication_fetcher.set_replication_distance_range(distance_256);
 
         let mut incoming_keys = Vec::new();
         let mut in_range_keys = 0;

ant-node-manager/Cargo.toml

@@ -43,7 +43,7 @@ colored = "2.0.4"
 color-eyre = "~0.6"
 dirs-next = "2.0.0"
 indicatif = { version = "0.17.5", features = ["tokio"] }
-libp2p = { git = "https://github.com/maqi/rust-libp2p.git", branch = "master", features = [] }
+libp2p = { version = "0.54.1", features = [] }
 libp2p-identity = { version = "0.2.7", features = ["rand"] }
 prost = { version = "0.9" }
 rand = "0.8.5"

ant-node-rpc-client/Cargo.toml

@@ -27,7 +27,7 @@ bls = { package = "blsttc", version = "8.0.1" }
 clap = { version = "4.2.1", features = ["derive"] }
 color-eyre = "0.6.2"
 hex = "~0.4.3"
-libp2p = { git = "https://github.com/maqi/rust-libp2p.git", branch = "master", features = ["kad"]}
+libp2p = { version = "0.54.1", features = ["kad"]}
 libp2p-identity = { version="0.2.7", features = ["rand"] }
 thiserror = "1.0.23"
 # # watch out updating this, protoc compiler needs to be installed on all build systems

ant-node/Cargo.toml

@@ -49,7 +49,7 @@ file-rotate = "0.7.3"
 futures = "~0.3.13"
 hex = "~0.4.3"
 itertools = "~0.12.1"
-libp2p = { git = "https://github.com/maqi/rust-libp2p.git", branch = "master", features = ["tokio", "dns", "kad", "macros"] }
+libp2p = { version = "0.54.1", features = ["tokio", "dns", "kad", "macros"] }
 num-traits = "0.2"
 prometheus-client = { version = "0.22", optional = true }
 # watch out updating this, protoc compiler needs to be installed on all build systems
@@ -83,7 +83,7 @@ walkdir = "~2.5.0"
 xor_name = "5.0.0"
 
 [dev-dependencies]
-ant-protocol = { path = "../ant-protocol", version = "0.17.15", features = ["rpc"]}
+ant-protocol = { path = "../ant-protocol", version = "0.17.15", features = ["rpc"] }
 assert_fs = "1.0.0"
 evmlib = { path = "../evmlib", version = "0.1.4" }
 autonomi = { path = "../autonomi", version = "0.2.4", features = ["registers"] }