[r2r] spv storage optimization #1

mm2src/coins/utxo/utxo_block_header_storage/indexedb_block_header_storage.rs

@@ -37,5 +37,5 @@ impl BlockHeaderStorageOps for IndexedDBBlockHeadersStorage {
 
     async fn get_block_height_by_hash(&self, _hash: H256) -> Result<Option<i64>, BlockHeaderStorageError> { Ok(None) }
 
-    async fn remove_headers_to_height(&self, _height: u64) -> Result<(), BlockHeaderStorageError> { Ok(()) }
+    async fn remove_headers_up_to_height(&self, _to_height: u64) -> Result<(), BlockHeaderStorageError> { Ok(()) }
 }

mm2src/coins/utxo/utxo_block_header_storage/mod.rs

@@ -98,7 +98,7 @@ impl BlockHeaderStorageOps for BlockHeaderStorage {
         self.inner.get_block_height_by_hash(hash).await
     }
 
-    async fn remove_headers_to_height(&self, to_height: u64) -> Result<(), BlockHeaderStorageError> {
-        self.inner.remove_headers_to_height(to_height).await
+    async fn remove_headers_up_to_height(&self, to_height: u64) -> Result<(), BlockHeaderStorageError> {
+        self.inner.remove_headers_up_to_height(to_height).await
     }
 }

mm2src/coins/utxo/utxo_block_header_storage/sql_block_header_storage.rs

@@ -87,9 +87,9 @@ fn get_block_height_by_hash(for_coin: &str) -> Result<String, BlockHeaderStorage
     Ok(sql)
 }
 
-fn remove_headers_to_height_sql(for_coin: &str, to_height: u64) -> Result<String, BlockHeaderStorageError> {
+fn remove_headers_up_to_height_sql(for_coin: &str, to_height: u64) -> Result<String, BlockHeaderStorageError> {
     let table_name = get_table_name_and_validate(for_coin)?;
-    let sql = format!("DELETE FROM {table_name} WHERE block_height < {to_height};");
+    let sql = format!("DELETE FROM {table_name} WHERE block_height <= {to_height};");
 
     Ok(sql)
 }
@@ -303,10 +303,10 @@ impl BlockHeaderStorageOps for SqliteBlockHeadersStorage {
         })
     }
 
-    async fn remove_headers_to_height(&self, to_height: u64) -> Result<(), BlockHeaderStorageError> {
+    async fn remove_headers_up_to_height(&self, to_height: u64) -> Result<(), BlockHeaderStorageError> {
         let coin = self.ticker.clone();
         let selfi = self.clone();
-        let sql = remove_headers_to_height_sql(&coin, to_height)?;
+        let sql = remove_headers_up_to_height_sql(&coin, to_height)?;
 
         async_blocking(move || {
             let conn = selfi.conn.lock().unwrap();
@@ -476,7 +476,7 @@ mod sql_block_headers_storage_tests {
     }
 
     #[test]
-    fn test_remove_headers_to_height() {
+    fn test_remove_headers_up_to_height() {
         let for_coin = "get";
         let storage = SqliteBlockHeadersStorage::in_memory(for_coin.into());
         let table = block_headers_cache_table(for_coin);
@@ -503,7 +503,7 @@ mod sql_block_headers_storage_tests {
         assert!(!storage.is_table_empty(&table));
 
         // Remove 2 headers from storage.
-        block_on(storage.remove_headers_to_height(201594)).unwrap();
+        block_on(storage.remove_headers_up_to_height(201594)).unwrap();
 
         // Validate that blockers 201593..201594 are removed from storage.
         for h in 201593..201594 {

mm2src/coins/utxo/utxo_builder/utxo_arc_builder.rs

@@ -319,7 +319,6 @@ pub(crate) async fn block_header_utxo_loop<T: UtxoCommonOps>(
 
         // Check if we need to max the number of headers to be stored in storage.
         if let Some(max_stored_block_headers) = spv_conf.max_stored_block_headers {
-            // to_block_height - 1(needed to keep previous header) headers will be removed from storage.
             if let Err(err) =
                 remove_excessive_headers_from_storage(storage, to_block_height, max_stored_block_headers).await
             {
@@ -428,7 +427,7 @@ async fn remove_excessive_headers_from_storage(
     let max_allowed_headers = max_allowed_headers.get();
     if last_height_to_be_added > max_allowed_headers {
         let height = last_height_to_be_added - max_allowed_headers;
-        return storage.remove_headers_to_height(height).await;
+        return storage.remove_headers_up_to_height(height).await;
     }
 
     Ok(())

mm2src/coins/utxo/utxo_tests.rs

@@ -4385,11 +4385,11 @@ fn test_block_header_utxo_loop() {
         assert!(expected_steps.lock().unwrap().is_empty());
 
         // Vlaidate max_stored_block_headers
-        // Since max_stored_block_headers == 1(used as previous height) + 15, headers from 2 - 3 shouldn't be in
+        // Since max_stored_block_headers15, headers from 2 - 5 shouldn't be in
         // storage anymore.
         for i in 2..=19 {
             let header = client.block_headers_storage().get_block_header(i).await.unwrap();
-            if i >= 4 {
+            if i >= 5 {
                 assert!(header.is_some());
                 break;
             }

mm2src/mm2_bitcoin/spv_validation/src/conf.rs

@@ -120,26 +120,28 @@ impl SPVConf {
         let rpc_header = SPVBlockHeader::from_block_header_and_height(rpc_header, height);
         let spv_header = &self.starting_block_header;
 
-        // Currently, BlockHeader::Compact is used in spv block header validation but some coins from rpc will return
+        // Currently, BlockHeader::Compact is used in spv block heazder validation but some coins from rpc will return
         // BlockHeader::U32, therefore, we need only the inner value in this case to validate the header bits.
         let rpc_header_bits: u32 = rpc_header.bits.into();
         let spv_header_bits: u32 = spv_header.bits.clone().into();
         if rpc_header_bits != spv_header_bits {
-            return Err(SPVError::SPVBlockHeaderError(
-                "Block header not acceptable - bad header bits".to_string(),
-            ));
+            return Err(SPVError::SPVBlockHeaderError(format!(
+                "Block header bits not acceptable - expected: {spv_header_bits} - found: {rpc_header_bits}"
+            )));
         };
 
         if rpc_header.hash != spv_header.hash {
-            return Err(SPVError::SPVBlockHeaderError(
-                "Block header not acceptable - bad header hash".to_string(),
-            ));
+            return Err(SPVError::SPVBlockHeaderError(format!(
+                "Block header hash not acceptable - expected: {} - found: {}",
+                spv_header.hash, rpc_header.hash,
+            )));
         };
 
         if rpc_header.time != spv_header.time {
-            return Err(SPVError::SPVBlockHeaderError(
-                "Block header not acceptable - bad header time".to_string(),
-            ));
+            return Err(SPVError::SPVBlockHeaderError(format!(
+                "Block header time not acceptable - expected: {} - found: {}",
+                spv_header.time, rpc_header.time,
+            )));
         };
 
         Ok(())

mm2src/mm2_bitcoin/spv_validation/src/storage.rs

@@ -81,5 +81,5 @@ pub trait BlockHeaderStorageOps: Send + Sync + 'static {
 
     async fn get_block_height_by_hash(&self, hash: H256) -> Result<Option<i64>, BlockHeaderStorageError>;
 
-    async fn remove_headers_to_height(&self, to_height: u64) -> Result<(), BlockHeaderStorageError>;
+    async fn remove_headers_up_to_height(&self, to_height: u64) -> Result<(), BlockHeaderStorageError>;
 }

mm2src/mm2_bitcoin/spv_validation/src/work.rs

@@ -246,7 +246,7 @@ pub(crate) mod tests {
             Ok(None)
         }
 
-        async fn remove_headers_to_height(&self, _height: u64) -> Result<(), BlockHeaderStorageError> { Ok(()) }
+        async fn remove_headers_up_to_height(&self, _height: u64) -> Result<(), BlockHeaderStorageError> { Ok(()) }
     }
 
     #[test]

mm2src/mm2_test_helpers/src/for_tests.rs

@@ -613,7 +613,7 @@ pub fn tbtc_with_spv_conf() -> Json {
             "starting_block_header": {
                 "height": 0,
                 "hash": "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943",
-                "bits": 487063544,
+                "bits": 486604799,
                 "time": 1296688602,
             },
             "validation_params": {